You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/12/06 01:42:00 UTC
[iotdb] branch lmh/OptSingleGroupby created (now 6f76c25c0b)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 6f76c25c0b make code more efficient
This branch includes the following new commits:
new b0dc02a392 spotless pom
new 45db7fd7e1 modify config
new 2bc25671aa tmp skip getPathModifications
new 39a59a9341 opt FunctionExpression constructor & remove old code
new 63e1a212d5 remove some time-consuming operation
new 90b171dbca add getAllSatisfiedDataWithoutModified
new 6f76c25c0b make code more efficient
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 03/07: tmp skip getPathModifications
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2bc25671aaef4b60acc692c2006bc834e4815819
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Dec 5 22:48:07 2022 +0800
tmp skip getPathModifications
---
.../iotdb/db/query/context/QueryContext.java | 43 +++++++++++-----------
1 file changed, 22 insertions(+), 21 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 60cffd8226..83dfeb478d 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -25,7 +25,6 @@ import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.metadata.path.PatternTreeMapFactory;
import org.apache.iotdb.db.metadata.path.PatternTreeMapFactory.ModsSerializer;
import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
@@ -94,26 +93,28 @@ public class QueryContext {
* them from 'modFile' and put then into the cache.
*/
public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) {
- // if the mods file does not exist, do not add it to the cache
- if (!modFile.exists()) {
- return Collections.emptyList();
- }
- Map<String, List<Modification>> fileModifications =
- filePathModCache.computeIfAbsent(modFile.getFilePath(), k -> new ConcurrentHashMap<>());
- return fileModifications.computeIfAbsent(
- path.getFullPath(),
- k -> {
- PatternTreeMap<Modification, ModsSerializer> allModifications =
- fileModCache.get(modFile.getFilePath());
- if (allModifications == null) {
- allModifications = PatternTreeMapFactory.getModsPatternTreeMap();
- for (Modification modification : modFile.getModifications()) {
- allModifications.append(modification.getPath(), modification);
- }
- fileModCache.put(modFile.getFilePath(), allModifications);
- }
- return sortAndMerge(allModifications.getOverlapped(path));
- });
+ return Collections.emptyList();
+ // // if the mods file does not exist, do not add it to the cache
+ // if (!modFile.exists()) {
+ // return Collections.emptyList();
+ // }
+ // Map<String, List<Modification>> fileModifications =
+ // filePathModCache.computeIfAbsent(modFile.getFilePath(), k -> new
+ // ConcurrentHashMap<>());
+ // return fileModifications.computeIfAbsent(
+ // path.getFullPath(),
+ // k -> {
+ // PatternTreeMap<Modification, ModsSerializer> allModifications =
+ // fileModCache.get(modFile.getFilePath());
+ // if (allModifications == null) {
+ // allModifications = PatternTreeMapFactory.getModsPatternTreeMap();
+ // for (Modification modification : modFile.getModifications()) {
+ // allModifications.append(modification.getPath(), modification);
+ // }
+ // fileModCache.put(modFile.getFilePath(), allModifications);
+ // }
+ // return sortAndMerge(allModifications.getOverlapped(path));
+ // });
}
private List<Modification> sortAndMerge(List<Modification> modifications) {
[iotdb] 01/07: spotless pom
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b0dc02a392e83cfc74f9f04d5c265e88a8aeebf3
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Dec 5 21:46:46 2022 +0800
spotless pom
---
client-cpp/pom.xml | 4 ++--
compile-tools/pom.xml | 6 +++---
distribution/pom.xml | 2 +-
example/client-cpp-example/pom.xml | 2 +-
example/trigger/pom.xml | 2 +-
example/udf/pom.xml | 2 +-
grafana-connector/pom.xml | 2 +-
grafana-plugin/pom.xml | 8 ++++----
integration/pom.xml | 6 +++---
jdbc/pom.xml | 2 +-
pom.xml | 8 ++++----
11 files changed, 22 insertions(+), 22 deletions(-)
diff --git a/client-cpp/pom.xml b/client-cpp/pom.xml
index 4324e2d2ad..2bb64eb3e3 100644
--- a/client-cpp/pom.xml
+++ b/client-cpp/pom.xml
@@ -121,8 +121,8 @@
<cmake.root.dir>${project.parent.basedir}/compile-tools/thrift/target/cmake-${cmake-version}-win64-x64/</cmake.root.dir>
<thrift.exec.absolute.path>${project.parent.basedir}/compile-tools/thrift/target/build/compiler/cpp/bin/${cmake.build.type}/thrift.exe</thrift.exec.absolute.path>
<iotdb.server.script>start-server.bat</iotdb.server.script>
- <boost.include.dir />
- <boost.library.dir />
+ <boost.include.dir/>
+ <boost.library.dir/>
</properties>
</profile>
<profile>
diff --git a/compile-tools/pom.xml b/compile-tools/pom.xml
index 434426856b..ba7ce8ad6f 100644
--- a/compile-tools/pom.xml
+++ b/compile-tools/pom.xml
@@ -35,7 +35,7 @@
<cmake-version>3.17.3</cmake-version>
<openssl.include.dir>-Dtrue1=true1</openssl.include.dir>
<bison.executable.dir>-Dtrue1=true1</bison.executable.dir>
- <cmake.build.type />
+ <cmake.build.type/>
</properties>
<modules>
<module>thrift</module>
@@ -138,8 +138,8 @@
<thrift.make.executable>make</thrift.make.executable>
<thrift.compiler.executable>thrift.exe</thrift.compiler.executable>
<gradlew.executable>gradlew.bat</gradlew.executable>
- <boost.include.dir />
- <boost.library.dir />
+ <boost.include.dir/>
+ <boost.library.dir/>
</properties>
</profile>
</profiles>
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 30d290f9d0..0041e6b27a 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -29,7 +29,7 @@
</parent>
<artifactId>iotdb-distribution</artifactId>
<name>IoTDB Distribution</name>
- <modules />
+ <modules/>
<build>
<plugins>
<plugin>
diff --git a/example/client-cpp-example/pom.xml b/example/client-cpp-example/pom.xml
index 95c59ce1e3..a55d9779d2 100644
--- a/example/client-cpp-example/pom.xml
+++ b/example/client-cpp-example/pom.xml
@@ -84,7 +84,7 @@
<properties>
<cmake.generator>Visual Studio 16 2019</cmake.generator>
<cmake.root.dir>${project.parent.basedir}/../compile-tools/thrift/target/cmake-${cmake-version}-win64-x64/</cmake.root.dir>
- <boost.include.dir />
+ <boost.include.dir/>
</properties>
</profile>
<profile>
diff --git a/example/trigger/pom.xml b/example/trigger/pom.xml
index c3cdf22bb1..d51aa63311 100644
--- a/example/trigger/pom.xml
+++ b/example/trigger/pom.xml
@@ -123,7 +123,7 @@
<importOrder>
<order>org.apache.iotdb,,javax,java,\#</order>
</importOrder>
- <removeUnusedImports />
+ <removeUnusedImports/>
</java>
</configuration>
<executions>
diff --git a/example/udf/pom.xml b/example/udf/pom.xml
index ffc0b0113a..37f81ec0da 100644
--- a/example/udf/pom.xml
+++ b/example/udf/pom.xml
@@ -117,7 +117,7 @@
<importOrder>
<order>org.apache.iotdb,,javax,java,\#</order>
</importOrder>
- <removeUnusedImports />
+ <removeUnusedImports/>
</java>
</configuration>
<executions>
diff --git a/grafana-connector/pom.xml b/grafana-connector/pom.xml
index 3a1d90beb2..1e29328142 100644
--- a/grafana-connector/pom.xml
+++ b/grafana-connector/pom.xml
@@ -170,7 +170,7 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>${start-class}</mainClass>
</transformer>
diff --git a/grafana-plugin/pom.xml b/grafana-plugin/pom.xml
index 02b6a02cc9..e22a5c4eb9 100644
--- a/grafana-plugin/pom.xml
+++ b/grafana-plugin/pom.xml
@@ -123,16 +123,16 @@
<configuration>
<tasks>
<condition property="osFamily" value="windows">
- <os family="windows" />
+ <os family="windows"/>
</condition>
<condition property="osFamily" value="unix">
- <os family="unix" />
+ <os family="unix"/>
</condition>
<exec executable="C:\\Windows\\System32\\cmd.exe" osfamily="windows">
- <arg line="/c backend-compile.bat" />
+ <arg line="/c backend-compile.bat"/>
</exec>
<exec executable="/bin/bash" osfamily="unix">
- <arg line="-c ./backend-compile.sh" />
+ <arg line="-c ./backend-compile.sh"/>
</exec>
</tasks>
</configuration>
diff --git a/integration/pom.xml b/integration/pom.xml
index a11052a101..dfa3c42eff 100644
--- a/integration/pom.xml
+++ b/integration/pom.xml
@@ -85,7 +85,7 @@
<id>LocalStandalone</id>
<properties>
<test.includedGroups>org.apache.iotdb.itbase.category.LocalStandaloneTest</test.includedGroups>
- <test.excludedGroups />
+ <test.excludedGroups/>
</properties>
<activation>
<activeByDefault>true</activeByDefault>
@@ -147,7 +147,7 @@
<id>Remote</id>
<properties>
<test.includedGroups>org.apache.iotdb.itbase.category.RemoteTest</test.includedGroups>
- <test.excludedGroups />
+ <test.excludedGroups/>
</properties>
<activation>
<activeByDefault>false</activeByDefault>
@@ -211,7 +211,7 @@
<id>Cluster</id>
<properties>
<test.includedGroups>org.apache.iotdb.itbase.category.ClusterTest</test.includedGroups>
- <test.excludedGroups />
+ <test.excludedGroups/>
</properties>
<activation>
<activeByDefault>false</activeByDefault>
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 3625b710a3..bdc9bf6de4 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -206,7 +206,7 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore />
+ <ignore/>
</action>
</pluginExecution>
</pluginExecutions>
diff --git a/pom.xml b/pom.xml
index ac65fc69f9..da93a06b8e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -172,7 +172,7 @@
<sonar.junit.reportPaths>target/surefire-reports,target/failsafe-reports</sonar.junit.reportPaths>
<!-- By default, the argLine is empty-->
<gson.version>2.8.9</gson.version>
- <argLine />
+ <argLine/>
<!-- whether enable compiling the cpp client-->
<client-cpp>false</client-cpp>
<!-- disable enforcer by default-->
@@ -800,7 +800,7 @@
<importOrder>
<order>org.apache.iotdb,,javax,java,\#</order>
</importOrder>
- <removeUnusedImports />
+ <removeUnusedImports/>
</java>
<lineEndings>UNIX</lineEndings>
</configuration>
@@ -878,7 +878,7 @@
<phase>validate</phase>
<configuration>
<rules>
- <dependencyConvergence />
+ <dependencyConvergence/>
</rules>
</configuration>
<goals>
@@ -924,7 +924,7 @@
</requireJavaVersion>
<!-- Disabled for now as it breaks the ability to build single modules -->
<!--reactorModuleConvergence/-->
- <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies" />
+ <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/>
</rules>
</configuration>
</execution>
[iotdb] 06/07: add getAllSatisfiedDataWithoutModified
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 90b171dbca2ca54884678a451121b5589457ff34
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Dec 6 09:36:15 2022 +0800
add getAllSatisfiedDataWithoutModified
---
.../tsfile/read/reader/page/ValuePageReader.java | 69 +++++++++++++++++++++-
1 file changed, 67 insertions(+), 2 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
index 45729ec05e..195b57f4d6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java
@@ -328,6 +328,67 @@ public class ValuePageReader {
}
}
+ public void writeColumnBuilderWithNextBatch(
+ long[] timeBatch, ColumnBuilder columnBuilder, boolean[] keepCurrentRow) {
+ if (valueBuffer == null) {
+ for (int i = 0, n = timeBatch.length; i < n; i++) {
+ if (keepCurrentRow[i]) {
+ columnBuilder.appendNull();
+ }
+ }
+ return;
+ }
+
+ for (int i = 0, n = timeBatch.length; i < n; i++) {
+ if (((bitmap[i / 8] & 0xFF) & (MASK >>> (i % 8))) == 0) {
+ if (keepCurrentRow[i]) {
+ columnBuilder.appendNull();
+ }
+ continue;
+ }
+ switch (dataType) {
+ case BOOLEAN:
+ boolean aBoolean = valueDecoder.readBoolean(valueBuffer);
+ if (keepCurrentRow[i]) {
+ columnBuilder.writeBoolean(aBoolean);
+ }
+ break;
+ case INT32:
+ int anInt = valueDecoder.readInt(valueBuffer);
+ if (keepCurrentRow[i]) {
+ columnBuilder.writeInt(anInt);
+ }
+ break;
+ case INT64:
+ long aLong = valueDecoder.readLong(valueBuffer);
+ if (keepCurrentRow[i]) {
+ columnBuilder.writeLong(aLong);
+ }
+ break;
+ case FLOAT:
+ float aFloat = valueDecoder.readFloat(valueBuffer);
+ if (keepCurrentRow[i]) {
+ columnBuilder.writeFloat(aFloat);
+ }
+ break;
+ case DOUBLE:
+ double aDouble = valueDecoder.readDouble(valueBuffer);
+ if (keepCurrentRow[i]) {
+ columnBuilder.writeDouble(aDouble);
+ }
+ break;
+ case TEXT:
+ Binary aBinary = valueDecoder.readBinary(valueBuffer);
+ if (keepCurrentRow[i]) {
+ columnBuilder.writeBinary(aBinary);
+ }
+ break;
+ default:
+ throw new UnSupportedDataTypeException(String.valueOf(dataType));
+ }
+ }
+ }
+
public Statistics getStatistics() {
return pageHeader.getStatistics();
}
@@ -358,8 +419,12 @@ public class ValuePageReader {
}
public void fillIsDeleted(long[] timestamp, boolean[] isDeleted) {
- for (int i = 0, n = timestamp.length; i < n; i++) {
- isDeleted[i] = isDeleted(timestamp[i]);
+ if (deleteIntervalList == null) {
+ Arrays.fill(isDeleted, false);
+ } else {
+ for (int i = 0, n = timestamp.length; i < n; i++) {
+ isDeleted[i] = isDeleted(timestamp[i]);
+ }
}
}
[iotdb] 02/07: modify config
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 45db7fd7e1ef7126c7c2c6a0851cd27f601f2ec8
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Dec 5 21:47:03 2022 +0800
modify config
---
.../java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java | 6 +++---
server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +++---
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 6f6fe727f8..597c644291 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -52,16 +52,16 @@ public class ConfigNodeConfig {
private int configNodeRegionId = 0;
/** ConfigNodeGroup consensus protocol */
- private String configNodeConsensusProtocolClass = ConsensusFactory.RATIS_CONSENSUS;
+ private String configNodeConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
/** Schema region consensus protocol */
- private String schemaRegionConsensusProtocolClass = ConsensusFactory.RATIS_CONSENSUS;
+ private String schemaRegionConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
/** Default number of SchemaRegion replicas */
private int schemaReplicationFactor = 1;
/** Data region consensus protocol */
- private String dataRegionConsensusProtocolClass = ConsensusFactory.IOT_CONSENSUS;
+ private String dataRegionConsensusProtocolClass = ConsensusFactory.SIMPLE_CONSENSUS;
/** Default number of DataRegion replicas */
private int dataReplicationFactor = 1;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 99dcf1b865..eac78863a3 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -397,13 +397,13 @@ public class IoTDBConfig {
private int avgSeriesPointNumberThreshold = 100000;
/** Enable inner space compaction for sequence files */
- private boolean enableSeqSpaceCompaction = true;
+ private boolean enableSeqSpaceCompaction = false;
/** Enable inner space compaction for unsequence files */
- private boolean enableUnseqSpaceCompaction = true;
+ private boolean enableUnseqSpaceCompaction = false;
/** Compact the unsequence files into the overlapped sequence files */
- private boolean enableCrossSpaceCompaction = true;
+ private boolean enableCrossSpaceCompaction = false;
/**
* The strategy of inner space compaction task. There are just one inner space compaction strategy
[iotdb] 04/07: opt FunctionExpression constructor & remove old code
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 39a59a9341ca63dd4fdb70bf976875980c958c78
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Dec 5 22:49:32 2022 +0800
opt FunctionExpression constructor & remove old code
---
.../iotdb/db/mpp/plan/expression/Expression.java | 30 --
.../iotdb/db/mpp/plan/expression/ResultColumn.java | 55 +---
.../plan/expression/binary/BinaryExpression.java | 101 ------
.../mpp/plan/expression/leaf/ConstantOperand.java | 20 --
.../db/mpp/plan/expression/leaf/NullOperand.java | 21 --
.../plan/expression/leaf/TimeSeriesOperand.java | 25 --
.../mpp/plan/expression/leaf/TimestampOperand.java | 26 --
.../plan/expression/multi/FunctionExpression.java | 97 +-----
.../plan/expression/ternary/TernaryExpression.java | 72 -----
.../mpp/plan/expression/unary/UnaryExpression.java | 41 ---
.../iotdb/db/qp/logical/crud/SelectComponent.java | 11 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 339 +--------------------
.../apache/iotdb/db/qp/utils/WildcardsRemover.java | 149 ---------
13 files changed, 14 insertions(+), 973 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/Expression.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/Expression.java
index 60c8d6364a..d9f572d441 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/Expression.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/Expression.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.mpp.plan.expression;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.binary.AdditionExpression;
import org.apache.iotdb.db.mpp.plan.expression.binary.DivisionExpression;
@@ -65,7 +63,6 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/** A skeleton class for expression */
public abstract class Expression extends StatementNode {
@@ -89,14 +86,6 @@ public abstract class Expression extends StatementNode {
return false;
}
- public boolean isUserDefinedAggregationFunctionExpression() {
- return false;
- }
-
- public boolean isTimeSeriesGeneratingFunctionExpression() {
- return false;
- }
-
public boolean isCompareBinaryExpression() {
return false;
}
@@ -120,25 +109,6 @@ public abstract class Expression extends StatementNode {
/** Sub-classes should override this method indicating if the expression is a constant operand */
protected abstract boolean isConstantOperandInternal();
- /////////////////////////////////////////////////////////////////////////////////////////////////
- // Operations for time series paths
- /////////////////////////////////////////////////////////////////////////////////////////////////
-
- // TODO: remove after MPP finish
- @Deprecated
- public abstract void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions);
-
- // TODO: remove after MPP finish
- @Deprecated
- public abstract void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions)
- throws LogicalOptimizeException;
-
- // TODO: remove after MPP finish
- @Deprecated
- public abstract void collectPaths(Set<PartialPath> pathSet);
-
/////////////////////////////////////////////////////////////////////////////////////////////////
// For UDF instances initialization
/////////////////////////////////////////////////////////////////////////////////////////////////
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ResultColumn.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ResultColumn.java
index 791da53e6e..19516f8d4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ResultColumn.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ResultColumn.java
@@ -20,16 +20,11 @@
package org.apache.iotdb.db.mpp.plan.expression;
import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
-import org.apache.iotdb.db.qp.utils.WildcardsRemover;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
/**
* {@code ResultColumn} is used to represent a result column of a query.
@@ -93,56 +88,8 @@ public class ResultColumn {
dataType = TSDataType.deserializeFrom(byteBuffer);
}
- /**
- * @param prefixPaths prefix paths in the from clause
- * @param resultColumns used to collect the result columns
- * @param needAliasCheck used to skip illegal alias judgement here. Including !isGroupByLevel
- * because count(*) may be * unfolded to more than one expression, but it still can be
- * aggregated together later.
- */
- public void concat(
- List<PartialPath> prefixPaths, List<ResultColumn> resultColumns, boolean needAliasCheck)
- throws LogicalOptimizeException {
- List<Expression> resultExpressions = new ArrayList<>();
- expression.concat(prefixPaths, resultExpressions);
- if (needAliasCheck && 1 < resultExpressions.size()) {
- throw new LogicalOptimizeException(
- String.format("alias '%s' can only be matched with one time series", alias));
- }
- for (Expression resultExpression : resultExpressions) {
- resultColumns.add(new ResultColumn(resultExpression, alias));
- }
- }
-
- /**
- * @param wildcardsRemover used to remove wildcards from {@code expression} and apply slimit &
- * soffset control
- * @param resultColumns used to collect the result columns
- * @param needAliasCheck used to skip illegal alias judgement here. Including !isGroupByLevel
- * because count(*) may be * unfolded to more than one expression, but it still can be
- * aggregated together later.
- */
- public void removeWildcards(
- WildcardsRemover wildcardsRemover, List<ResultColumn> resultColumns, boolean needAliasCheck)
- throws LogicalOptimizeException {
- List<Expression> resultExpressions = new ArrayList<>();
- expression.removeWildcards(wildcardsRemover, resultExpressions);
- if (needAliasCheck && 1 < resultExpressions.size()) {
- throw new LogicalOptimizeException(
- String.format("alias '%s' can only be matched with one time series", alias));
- }
- for (Expression resultExpression : resultExpressions) {
- resultColumns.add(new ResultColumn(resultExpression, alias));
- }
- }
-
public List<PartialPath> collectPaths() {
- if (allPathsInExpression == null) {
- Set<PartialPath> pathSet = new HashSet<>();
- expression.collectPaths(pathSet);
- allPathsInExpression = new ArrayList<>(pathSet);
- }
- return allPathsInExpression;
+ throw new UnsupportedOperationException("");
}
public Expression getExpression() {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/binary/BinaryExpression.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/binary/BinaryExpression.java
index 58722f7da5..8c6e5b68ce 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/binary/BinaryExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/binary/BinaryExpression.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.mpp.plan.expression.binary;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
@@ -34,11 +32,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.ZoneId;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Set;
public abstract class BinaryExpression extends Expression {
@@ -81,108 +77,11 @@ public abstract class BinaryExpression extends Expression {
return leftExpression.isConstantOperand() && rightExpression.isConstantOperand();
}
- @Override
- public boolean isTimeSeriesGeneratingFunctionExpression() {
- return !isUserDefinedAggregationFunctionExpression();
- }
-
- @Override
- public boolean isUserDefinedAggregationFunctionExpression() {
- return leftExpression.isBuiltInAggregationFunctionExpression()
- || rightExpression.isBuiltInAggregationFunctionExpression()
- || leftExpression.isUserDefinedAggregationFunctionExpression()
- || rightExpression.isUserDefinedAggregationFunctionExpression();
- }
-
@Override
public List<Expression> getExpressions() {
return Arrays.asList(leftExpression, rightExpression);
}
- @Override
- public final void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- List<Expression> leftExpressions = new ArrayList<>();
- leftExpression.concat(prefixPaths, leftExpressions);
-
- List<Expression> rightExpressions = new ArrayList<>();
- rightExpression.concat(prefixPaths, rightExpressions);
-
- reconstruct(leftExpressions, rightExpressions, resultExpressions);
- }
-
- @Override
- public final void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- List<Expression> leftExpressions = new ArrayList<>();
- leftExpression.removeWildcards(wildcardsRemover, leftExpressions);
-
- List<Expression> rightExpressions = new ArrayList<>();
- rightExpression.removeWildcards(wildcardsRemover, rightExpressions);
-
- reconstruct(leftExpressions, rightExpressions, resultExpressions);
- }
-
- private void reconstruct(
- List<Expression> leftExpressions,
- List<Expression> rightExpressions,
- List<Expression> resultExpressions) {
- for (Expression le : leftExpressions) {
- for (Expression re : rightExpressions) {
- switch (operator()) {
- case "+":
- resultExpressions.add(new AdditionExpression(le, re));
- break;
- case "-":
- resultExpressions.add(new SubtractionExpression(le, re));
- break;
- case "*":
- resultExpressions.add(new MultiplicationExpression(le, re));
- break;
- case "/":
- resultExpressions.add(new DivisionExpression(le, re));
- break;
- case "%":
- resultExpressions.add(new ModuloExpression(le, re));
- break;
- case "<":
- resultExpressions.add(new LessThanExpression(le, re));
- break;
- case "<=":
- resultExpressions.add(new LessEqualExpression(le, re));
- break;
- case ">":
- resultExpressions.add(new GreaterThanExpression(le, re));
- break;
- case ">=":
- resultExpressions.add(new GreaterEqualExpression(le, re));
- break;
- case "=":
- resultExpressions.add(new EqualToExpression(le, re));
- break;
- case "!=":
- resultExpressions.add(new NonEqualExpression(le, re));
- break;
- case "&":
- resultExpressions.add(new LogicAndExpression(le, re));
- break;
- case "|":
- resultExpressions.add(new LogicOrExpression(le, re));
- break;
- default:
- throw new UnsupportedOperationException();
- }
- }
- }
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- leftExpression.collectPaths(pathSet);
- rightExpression.collectPaths(pathSet);
- }
-
@Override
public void constructUdfExecutors(
Map<String, UDTFExecutor> expressionName2Executor, ZoneId zoneId) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/ConstantOperand.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/ConstantOperand.java
index 592c339f7b..9f7920115b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/ConstantOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/ConstantOperand.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.mpp.plan.expression.leaf;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.InputLocation;
@@ -36,7 +34,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/** Constant operand */
public class ConstantOperand extends LeafOperand {
@@ -78,23 +75,6 @@ public class ConstantOperand extends LeafOperand {
return true;
}
- @Override
- public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- resultExpressions.add(this);
- }
-
- @Override
- public void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions) {
- resultExpressions.add(this);
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- // Do nothing
- }
-
@Override
public void bindInputLayerColumnIndexWithExpression(UDTFPlan udtfPlan) {
// Do nothing
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/NullOperand.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/NullOperand.java
index e157ddcbeb..02bfcfcee4 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/NullOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/NullOperand.java
@@ -19,22 +19,17 @@
package org.apache.iotdb.db.mpp.plan.expression.leaf;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
-import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.InputLocation;
import org.apache.iotdb.db.mpp.transformation.dag.memory.LayerMemoryAssigner;
import org.apache.iotdb.db.qp.physical.crud.UDTFPlan;
-import org.apache.iotdb.db.qp.utils.WildcardsRemover;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import java.util.Set;
public class NullOperand extends LeafOperand {
@Override
@@ -52,22 +47,6 @@ public class NullOperand extends LeafOperand {
return true;
}
- @Override
- public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- resultExpressions.add(this);
- }
-
- @Override
- public void removeWildcards(WildcardsRemover wildcardsRemover, List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- resultExpressions.add(this);
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- // do nothing
- }
-
@Override
public void bindInputLayerColumnIndexWithExpression(UDTFPlan udtfPlan) {
// do nothing
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimeSeriesOperand.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimeSeriesOperand.java
index 2f671dd8cf..5c3682b783 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimeSeriesOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimeSeriesOperand.java
@@ -22,8 +22,6 @@ package org.apache.iotdb.db.mpp.plan.expression.leaf;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
-import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.InputLocation;
@@ -36,7 +34,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import java.util.Set;
public class TimeSeriesOperand extends LeafOperand {
@@ -75,28 +72,6 @@ public class TimeSeriesOperand extends LeafOperand {
return false;
}
- @Override
- public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- for (PartialPath prefixPath : prefixPaths) {
- resultExpressions.add(new TimeSeriesOperand(prefixPath.concatPath(path)));
- }
- }
-
- @Override
- public void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- for (PartialPath actualPath : wildcardsRemover.removeWildcardFrom(path)) {
- resultExpressions.add(new TimeSeriesOperand(actualPath));
- }
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- pathSet.add(path);
- }
-
@Override
public void bindInputLayerColumnIndexWithExpression(UDTFPlan udtfPlan) {
inputColumnIndex = udtfPlan.getReaderIndexByExpressionName(toString());
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimestampOperand.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimestampOperand.java
index 496b599aa9..326e2c1d3b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimestampOperand.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/leaf/TimestampOperand.java
@@ -20,8 +20,6 @@
package org.apache.iotdb.db.mpp.plan.expression.leaf;
import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
-import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.InputLocation;
@@ -33,7 +31,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
-import java.util.Set;
public class TimestampOperand extends LeafOperand {
@@ -52,29 +49,6 @@ public class TimestampOperand extends LeafOperand {
return visitor.visitTimeStampOperand(this, context);
}
- @Override
- public boolean isTimeSeriesGeneratingFunctionExpression() {
- return true;
- }
-
- @Override
- public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- resultExpressions.add(this);
- }
-
- @Override
- public void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- resultExpressions.add(this);
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- pathSet.add(TIMESTAMP_PARTIAL_PATH);
- }
-
@Override
public void bindInputLayerColumnIndexWithExpression(UDTFPlan udtfPlan) {
// do nothing
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
index 33694b6d95..3cf34158eb 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/multi/FunctionExpression.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.mpp.plan.expression.multi;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.ExpressionType;
@@ -33,7 +32,6 @@ import org.apache.iotdb.db.mpp.transformation.dag.memory.LayerMemoryAssigner;
import org.apache.iotdb.db.mpp.transformation.dag.udf.UDTFExecutor;
import org.apache.iotdb.db.mpp.transformation.dag.udf.UDTFInformationInferrer;
import org.apache.iotdb.db.qp.physical.crud.UDTFPlan;
-import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.udf.api.customizer.strategy.AccessStrategy;
@@ -48,7 +46,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.stream.Collectors;
public class FunctionExpression extends Expression {
@@ -57,9 +54,7 @@ public class FunctionExpression extends Expression {
* true: aggregation function<br>
* false: time series generating function
*/
- private final boolean isBuiltInAggregationFunctionExpression;
-
- private boolean isUserDefinedAggregationFunctionExpression;
+ private Boolean isBuiltInAggregationFunctionExpressionCache;
private final String functionName;
private final LinkedHashMap<String, String> functionAttributes;
@@ -79,10 +74,6 @@ public class FunctionExpression extends Expression {
this.functionName = functionName;
functionAttributes = new LinkedHashMap<>();
expressions = new ArrayList<>();
-
- isBuiltInAggregationFunctionExpression =
- BuiltinAggregationFunction.getNativeFunctionNames().contains(functionName.toLowerCase());
- isConstantOperandCache = true;
}
public FunctionExpression(
@@ -92,16 +83,6 @@ public class FunctionExpression extends Expression {
this.functionName = functionName;
this.functionAttributes = functionAttributes;
this.expressions = expressions;
-
- isBuiltInAggregationFunctionExpression =
- BuiltinAggregationFunction.getNativeFunctionNames().contains(functionName.toLowerCase());
- isConstantOperandCache = expressions.stream().anyMatch(Expression::isConstantOperand);
- isUserDefinedAggregationFunctionExpression =
- expressions.stream()
- .anyMatch(
- v ->
- v.isUserDefinedAggregationFunctionExpression()
- || v.isBuiltInAggregationFunctionExpression());
}
public FunctionExpression(ByteBuffer byteBuffer) {
@@ -114,16 +95,6 @@ public class FunctionExpression extends Expression {
for (int i = 0; i < expressionSize; i++) {
expressions.add(Expression.deserialize(byteBuffer));
}
-
- isBuiltInAggregationFunctionExpression =
- BuiltinAggregationFunction.getNativeFunctionNames().contains(functionName);
- isConstantOperandCache = expressions.stream().anyMatch(Expression::isConstantOperand);
- isUserDefinedAggregationFunctionExpression =
- expressions.stream()
- .anyMatch(
- v ->
- v.isUserDefinedAggregationFunctionExpression()
- || v.isBuiltInAggregationFunctionExpression());
}
@Override
@@ -133,27 +104,23 @@ public class FunctionExpression extends Expression {
@Override
public boolean isBuiltInAggregationFunctionExpression() {
- return isBuiltInAggregationFunctionExpression;
+ if (isBuiltInAggregationFunctionExpressionCache == null) {
+ isBuiltInAggregationFunctionExpressionCache =
+ BuiltinAggregationFunction.getNativeFunctionNames().contains(functionName.toLowerCase());
+ }
+ return isBuiltInAggregationFunctionExpressionCache;
}
@Override
public boolean isConstantOperandInternal() {
+ if (isConstantOperandCache == null) {
+ isConstantOperandCache = expressions.stream().anyMatch(Expression::isConstantOperand);
+ }
return isConstantOperandCache;
}
- @Override
- public boolean isTimeSeriesGeneratingFunctionExpression() {
- return !isBuiltInAggregationFunctionExpression()
- && !isUserDefinedAggregationFunctionExpression();
- }
-
- @Override
- public boolean isUserDefinedAggregationFunctionExpression() {
- return isUserDefinedAggregationFunctionExpression;
- }
-
public boolean isCountStar() {
- if (!isBuiltInAggregationFunctionExpression) {
+ if (!isBuiltInAggregationFunctionExpression()) {
return false;
}
return getPaths().size() == 1
@@ -168,11 +135,6 @@ public class FunctionExpression extends Expression {
}
public void addExpression(Expression expression) {
- isConstantOperandCache = isConstantOperandCache && expression.isConstantOperand();
- isUserDefinedAggregationFunctionExpression =
- isUserDefinedAggregationFunctionExpression
- || expression.isUserDefinedAggregationFunctionExpression()
- || expression.isBuiltInAggregationFunctionExpression();
expressions.add(expression);
}
@@ -193,43 +155,6 @@ public class FunctionExpression extends Expression {
return expressions;
}
- @Override
- public void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- List<List<Expression>> resultExpressionsForRecursionList = new ArrayList<>();
-
- for (Expression suffixExpression : expressions) {
- List<Expression> resultExpressionsForRecursion = new ArrayList<>();
- suffixExpression.concat(prefixPaths, resultExpressionsForRecursion);
- resultExpressionsForRecursionList.add(resultExpressionsForRecursion);
- }
-
- List<List<Expression>> functionExpressions = new ArrayList<>();
- ConcatPathOptimizer.cartesianProduct(
- resultExpressionsForRecursionList, functionExpressions, 0, new ArrayList<>());
- for (List<Expression> functionExpression : functionExpressions) {
- resultExpressions.add(
- new FunctionExpression(functionName, functionAttributes, functionExpression));
- }
- }
-
- @Override
- public void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- for (List<Expression> functionExpression : wildcardsRemover.removeWildcardsFrom(expressions)) {
- resultExpressions.add(
- new FunctionExpression(functionName, functionAttributes, functionExpression));
- }
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- for (Expression expression : expressions) {
- expression.collectPaths(pathSet);
- }
- }
-
@Override
public void constructUdfExecutors(
Map<String, UDTFExecutor> expressionName2Executor, ZoneId zoneId) {
@@ -275,7 +200,7 @@ public class FunctionExpression extends Expression {
@Override
public boolean isMappable(Map<NodeRef<Expression>, TSDataType> expressionTypes) {
- if (isBuiltInAggregationFunctionExpression) {
+ if (isBuiltInAggregationFunctionExpression()) {
return true;
}
return new UDTFInformationInferrer(functionName)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ternary/TernaryExpression.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ternary/TernaryExpression.java
index b37ad2520c..037c084769 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ternary/TernaryExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/ternary/TernaryExpression.java
@@ -21,8 +21,6 @@
package org.apache.iotdb.db.mpp.plan.expression.ternary;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
@@ -36,11 +34,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.ZoneId;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import java.util.Set;
public abstract class TernaryExpression extends Expression {
protected final Expression firstExpression;
@@ -85,79 +81,11 @@ public abstract class TernaryExpression extends Expression {
&& thirdExpression.isConstantOperand();
}
- @Override
- public boolean isTimeSeriesGeneratingFunctionExpression() {
- return !isUserDefinedAggregationFunctionExpression();
- }
-
- @Override
- public boolean isUserDefinedAggregationFunctionExpression() {
- return firstExpression.isBuiltInAggregationFunctionExpression()
- || secondExpression.isBuiltInAggregationFunctionExpression()
- || thirdExpression.isUserDefinedAggregationFunctionExpression();
- }
-
@Override
public List<Expression> getExpressions() {
return Arrays.asList(firstExpression, secondExpression, thirdExpression);
}
- @Override
- public final void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- List<Expression> firstExpressions = new ArrayList<>();
- firstExpression.concat(prefixPaths, firstExpressions);
-
- List<Expression> secondExpressions = new ArrayList<>();
- secondExpression.concat(prefixPaths, secondExpressions);
-
- List<Expression> thirdExpressions = new ArrayList<>();
- secondExpression.concat(prefixPaths, thirdExpressions);
-
- reconstruct(firstExpressions, secondExpressions, thirdExpressions, resultExpressions);
- }
-
- @Override
- public final void removeWildcards(
- org.apache.iotdb.db.qp.utils.WildcardsRemover wildcardsRemover,
- List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- List<Expression> firstExpressions = new ArrayList<>();
- firstExpression.removeWildcards(wildcardsRemover, firstExpressions);
-
- List<Expression> secondExpressions = new ArrayList<>();
- secondExpression.removeWildcards(wildcardsRemover, secondExpressions);
-
- List<Expression> thirdExpressions = new ArrayList<>();
- thirdExpression.removeWildcards(wildcardsRemover, secondExpressions);
- reconstruct(firstExpressions, secondExpressions, thirdExpressions, resultExpressions);
- }
-
- private void reconstruct(
- List<Expression> firstExpressions,
- List<Expression> secondExpressions,
- List<Expression> thirdExpressions,
- List<Expression> resultExpressions) {
- for (Expression fe : firstExpressions) {
- for (Expression se : secondExpressions)
- for (Expression te : thirdExpressions) {
- switch (operator()) {
- case "between":
- resultExpressions.add(new BetweenExpression(fe, se, te));
- break;
- default:
- throw new UnsupportedOperationException();
- }
- }
- }
- }
-
- @Override
- public void collectPaths(Set<PartialPath> pathSet) {
- firstExpression.collectPaths(pathSet);
- secondExpression.collectPaths(pathSet);
- thirdExpression.collectPaths(pathSet);
- }
-
@Override
public boolean isMappable(Map<NodeRef<Expression>, TSDataType> expressionTypes) {
return firstExpression.isMappable(expressionTypes)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/unary/UnaryExpression.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/unary/UnaryExpression.java
index f006a32350..c0520453af 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/unary/UnaryExpression.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/expression/unary/UnaryExpression.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.mpp.plan.expression.unary;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.mpp.common.NodeRef;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
import org.apache.iotdb.db.mpp.plan.expression.visitor.ExpressionVisitor;
@@ -28,18 +26,15 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.InputLocation;
import org.apache.iotdb.db.mpp.transformation.dag.memory.LayerMemoryAssigner;
import org.apache.iotdb.db.mpp.transformation.dag.udf.UDTFExecutor;
import org.apache.iotdb.db.qp.physical.crud.UDTFPlan;
-import org.apache.iotdb.db.qp.utils.WildcardsRemover;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.time.ZoneId;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
public abstract class UnaryExpression extends Expression {
@@ -68,22 +63,6 @@ public abstract class UnaryExpression extends Expression {
return Collections.singletonList(expression);
}
- @Override
- public final boolean isTimeSeriesGeneratingFunctionExpression() {
- return !isUserDefinedAggregationFunctionExpression();
- }
-
- @Override
- public final boolean isUserDefinedAggregationFunctionExpression() {
- return expression.isUserDefinedAggregationFunctionExpression()
- || expression.isBuiltInAggregationFunctionExpression();
- }
-
- @Override
- public final void collectPaths(Set<PartialPath> pathSet) {
- expression.collectPaths(pathSet);
- }
-
@Override
public final void constructUdfExecutors(
Map<String, UDTFExecutor> expressionName2Executor, ZoneId zoneId) {
@@ -118,26 +97,6 @@ public abstract class UnaryExpression extends Expression {
return expression.isMappable(expressionTypes);
}
- @Override
- public final void concat(List<PartialPath> prefixPaths, List<Expression> resultExpressions) {
- List<Expression> resultExpressionsForRecursion = new ArrayList<>();
- expression.concat(prefixPaths, resultExpressionsForRecursion);
- for (Expression resultExpression : resultExpressionsForRecursion) {
- resultExpressions.add(constructExpression(resultExpression));
- }
- }
-
- @Override
- public final void removeWildcards(
- WildcardsRemover wildcardsRemover, List<Expression> resultExpressions)
- throws LogicalOptimizeException {
- List<Expression> resultExpressionsForRecursion = new ArrayList<>();
- expression.removeWildcards(wildcardsRemover, resultExpressionsForRecursion);
- for (Expression resultExpression : resultExpressionsForRecursion) {
- resultExpressions.add(constructExpression(resultExpression));
- }
- }
-
protected abstract Expression constructExpression(Expression childExpression);
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectComponent.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectComponent.java
index 2b4616278e..fb494433b9 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectComponent.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/crud/SelectComponent.java
@@ -82,16 +82,7 @@ public class SelectComponent {
}
public void addResultColumn(ResultColumn resultColumn) {
- resultColumns.add(resultColumn);
- if (resultColumn.getExpression().isUserDefinedAggregationFunctionExpression()) {
- hasUserDefinedAggregationFunction = true;
- }
- if (resultColumn.getExpression().isBuiltInAggregationFunctionExpression()) {
- hasPlainAggregationFunction = true;
- }
- if (resultColumn.getExpression().isTimeSeriesGeneratingFunctionExpression()) {
- hasTimeSeriesGeneratingFunction = true;
- }
+ throw new UnsupportedOperationException("");
}
public void setResultColumns(List<ResultColumn> resultColumns) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
index 68fd6ed86e..233aa34ab3 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/optimizer/ConcatPathOptimizer.java
@@ -18,42 +18,15 @@
*/
package org.apache.iotdb.db.qp.strategy.optimizer;
-import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.commons.path.MeasurementPath;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
-import org.apache.iotdb.db.exception.sql.SQLParserException;
-import org.apache.iotdb.db.mpp.plan.expression.Expression;
-import org.apache.iotdb.db.mpp.plan.expression.ResultColumn;
-import org.apache.iotdb.db.mpp.plan.expression.leaf.TimeSeriesOperand;
-import org.apache.iotdb.db.qp.constant.FilterConstant.FilterType;
-import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.logical.Operator;
-import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator;
-import org.apache.iotdb.db.qp.logical.crud.FilterOperator;
-import org.apache.iotdb.db.qp.logical.crud.FromComponent;
-import org.apache.iotdb.db.qp.logical.crud.FunctionOperator;
-import org.apache.iotdb.db.qp.logical.crud.InOperator;
-import org.apache.iotdb.db.qp.logical.crud.LikeOperator;
-import org.apache.iotdb.db.qp.logical.crud.QueryOperator;
-import org.apache.iotdb.db.qp.logical.crud.RegexpOperator;
-import org.apache.iotdb.db.qp.logical.crud.SelectComponent;
-import org.apache.iotdb.db.qp.logical.crud.WhereComponent;
-import org.apache.iotdb.db.qp.utils.GroupByLevelController;
-import org.apache.iotdb.db.qp.utils.WildcardsRemover;
-import org.apache.iotdb.db.service.IoTDB;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
-import java.util.Set;
/** concat paths in select and from clause. */
public class ConcatPathOptimizer implements ILogicalOptimizer {
@@ -68,317 +41,7 @@ public class ConcatPathOptimizer implements ILogicalOptimizer {
@Override
public Operator transform(Operator operator)
throws LogicalOptimizeException, PathNumOverLimitException {
- QueryOperator queryOperator = (QueryOperator) operator;
- if (!optimizable(queryOperator)) {
- return queryOperator;
- }
-
- concatSelect(queryOperator);
- concatWithoutNullColumns(queryOperator);
- removeWildcardsInSelectPaths(queryOperator);
- removeWildcardsWithoutNullColumns(queryOperator);
- concatFilterAndRemoveWildcards(queryOperator);
- return queryOperator;
- }
-
- private boolean optimizable(QueryOperator queryOperator) {
- if (queryOperator.isAlignByDevice()) {
- return false;
- }
-
- SelectComponent select = queryOperator.getSelectComponent();
- if (select == null || select.getResultColumns().isEmpty()) {
- LOGGER.warn(WARNING_NO_SUFFIX_PATHS);
- return false;
- }
-
- FromComponent from = queryOperator.getFromComponent();
- if (from == null || from.getPrefixPaths().isEmpty()) {
- LOGGER.warn(WARNING_NO_PREFIX_PATHS);
- return false;
- }
-
- return true;
- }
-
- private void concatSelect(QueryOperator queryOperator) throws LogicalOptimizeException {
- List<PartialPath> prefixPaths = queryOperator.getFromComponent().getPrefixPaths();
- List<ResultColumn> resultColumns = new ArrayList<>();
- for (ResultColumn suffixColumn : queryOperator.getSelectComponent().getResultColumns()) {
- boolean needAliasCheck = suffixColumn.hasAlias() && !queryOperator.isGroupByLevel();
- suffixColumn.concat(prefixPaths, resultColumns, needAliasCheck);
- }
- queryOperator.getSelectComponent().setResultColumns(resultColumns);
- }
-
- private void concatWithoutNullColumns(QueryOperator queryOperator)
- throws LogicalOptimizeException {
- List<PartialPath> prefixPaths = queryOperator.getFromComponent().getPrefixPaths();
- // has without null columns
- if (queryOperator.getSpecialClauseComponent() != null
- && !queryOperator.getSpecialClauseComponent().getWithoutNullColumns().isEmpty()) {
- List<Expression> withoutNullColumns = new ArrayList<>();
- for (Expression expression :
- queryOperator.getSpecialClauseComponent().getWithoutNullColumns()) {
- concatWithoutNullColumns(
- prefixPaths, expression, withoutNullColumns, queryOperator.getAliasSet());
- }
- queryOperator.getSpecialClauseComponent().setWithoutNullColumns(withoutNullColumns);
- }
- }
-
- private void concatWithoutNullColumns(
- List<PartialPath> prefixPaths,
- Expression expression,
- List<Expression> withoutNullColumns,
- Set<String> aliasSet)
- throws LogicalOptimizeException {
- if (expression instanceof TimeSeriesOperand) {
- TimeSeriesOperand timeSeriesOperand = (TimeSeriesOperand) expression;
- if (timeSeriesOperand
- .getPath()
- .getFullPath()
- .startsWith(SQLConstant.ROOT + ".")) { // start with "root." don't concat
- // because the full path that starts with 'root.' won't be split
- // so we need to split it.
- if (((TimeSeriesOperand) expression).getPath().getNodeLength() == 1) { // no split
- try {
- ((TimeSeriesOperand) expression)
- .setPath(
- new PartialPath(
- PathUtils.splitPathToDetachedNodes(
- ((TimeSeriesOperand) expression)
- .getPath()
- .getFirstNode()))); // split path To nodes
- } catch (IllegalPathException e) {
- throw new LogicalOptimizeException(e.getMessage());
- }
- }
- withoutNullColumns.add(expression);
- } else {
- if (!aliasSet.contains(expression.getExpressionString())) { // not alias, concat
- List<Expression> resultExpressions = new ArrayList<>();
- expression.concat(prefixPaths, resultExpressions);
- withoutNullColumns.addAll(resultExpressions);
- } else { // alias, don't concat
- withoutNullColumns.add(expression);
- }
- }
- } else {
- List<Expression> resultExpressions = new ArrayList<>();
- expression.concat(prefixPaths, resultExpressions);
- withoutNullColumns.addAll(resultExpressions);
- }
- }
-
- private void removeWildcardsInSelectPaths(QueryOperator queryOperator)
- throws LogicalOptimizeException, PathNumOverLimitException {
- if (queryOperator.getIndexType() != null) {
- return;
- }
-
- List<ResultColumn> resultColumns = new ArrayList<>();
- // Only used for group by level
- GroupByLevelController groupByLevelController = null;
- if (queryOperator.isGroupByLevel()) {
- groupByLevelController = new GroupByLevelController(queryOperator);
- queryOperator.resetSLimitOffset();
- resultColumns = new LinkedList<>();
- }
-
- WildcardsRemover wildcardsRemover = new WildcardsRemover(queryOperator);
- for (ResultColumn resultColumn : queryOperator.getSelectComponent().getResultColumns()) {
- boolean needAliasCheck = resultColumn.hasAlias() && !queryOperator.isGroupByLevel();
- resultColumn.removeWildcards(wildcardsRemover, resultColumns, needAliasCheck);
- if (groupByLevelController != null) {
- groupByLevelController.control(resultColumn, resultColumns);
- }
- if (wildcardsRemover.checkIfPathNumberIsOverLimit(resultColumns)) {
- break;
- }
- }
- wildcardsRemover.checkIfSoffsetIsExceeded(resultColumns);
- queryOperator.getSelectComponent().setResultColumns(resultColumns);
- if (groupByLevelController != null) {
- queryOperator.getSpecialClauseComponent().setGroupByLevelController(groupByLevelController);
- }
- }
-
- private void removeWildcardsWithoutNullColumns(QueryOperator queryOperator)
- throws LogicalOptimizeException {
- if (queryOperator.getIndexType() != null) {
- return;
- }
-
- if (queryOperator.getSpecialClauseComponent() == null) {
- return;
- }
-
- List<Expression> expressions =
- queryOperator.getSpecialClauseComponent().getWithoutNullColumns();
- WildcardsRemover withoutNullWildcardsRemover = new WildcardsRemover(queryOperator);
-
- // because timeSeries path may be with "*", so need to remove it for getting some actual
- // timeSeries paths
- // actualExpressions store the actual timeSeries paths
- List<Expression> actualExpressions = new ArrayList<>();
- List<Expression> resultExpressions = new ArrayList<>();
-
- // because expression.removeWildcards will ignore the TimeSeries path that exists in the meta
- // so we need to recognise the alias, just simply add to the resultExpressions
- for (Expression expression : expressions) {
- if (queryOperator.getAliasSet().contains(expression.getExpressionString())) {
- resultExpressions.add(expression);
- continue;
- }
- expression.removeWildcards(withoutNullWildcardsRemover, actualExpressions);
- }
-
- // group by level, use groupedPathMap
- GroupByLevelController groupByLevelController =
- queryOperator.getSpecialClauseComponent().getGroupByLevelController();
- if (groupByLevelController != null) {
- for (Expression expression : actualExpressions) {
- String groupedPath =
- groupByLevelController.getGroupedPath(expression.getExpressionString());
- if (groupedPath != null) {
- resultExpressions.add(new TimeSeriesOperand(new PartialPath(groupedPath, false)));
- } else {
- resultExpressions.add(expression);
- }
- }
- } else {
- resultExpressions.addAll(actualExpressions);
- }
- queryOperator.getSpecialClauseComponent().setWithoutNullColumns(resultExpressions);
- }
-
- private void concatFilterAndRemoveWildcards(QueryOperator queryOperator)
- throws LogicalOptimizeException {
- WhereComponent whereComponent = queryOperator.getWhereComponent();
- if (whereComponent == null) {
- return;
- }
-
- Set<PartialPath> filterPaths = new HashSet<>();
- whereComponent.setFilterOperator(
- concatFilterAndRemoveWildcards(
- queryOperator.getFromComponent().getPrefixPaths(),
- whereComponent.getFilterOperator(),
- filterPaths,
- queryOperator.isPrefixMatchPath()));
- whereComponent.getFilterOperator().setPathSet(filterPaths);
- }
-
- private FilterOperator concatFilterAndRemoveWildcards(
- List<PartialPath> fromPaths,
- FilterOperator operator,
- Set<PartialPath> filterPaths,
- boolean isPrefixMatch)
- throws LogicalOptimizeException {
- if (!operator.isLeaf()) {
- List<FilterOperator> newFilterList = new ArrayList<>();
- for (FilterOperator child : operator.getChildren()) {
- newFilterList.add(
- concatFilterAndRemoveWildcards(fromPaths, child, filterPaths, isPrefixMatch));
- }
- operator.setChildren(newFilterList);
- return operator;
- }
- FunctionOperator functionOperator = (FunctionOperator) operator;
- PartialPath filterPath = functionOperator.getSinglePath();
- List<PartialPath> concatPaths = new ArrayList<>();
- if (SQLConstant.isReservedPath(filterPath)) {
- // do nothing in the case of "where time > 5"
- filterPaths.add(filterPath);
- return operator;
- } else if (filterPath.getFirstNode().startsWith(SQLConstant.ROOT)) {
- // do nothing in the case of "where root.d1.s1 > 5"
- concatPaths.add(filterPath);
- } else {
- fromPaths.forEach(fromPath -> concatPaths.add(fromPath.concatPath(filterPath)));
- }
-
- List<PartialPath> noStarPaths = removeWildcardsInConcatPaths(concatPaths, isPrefixMatch);
- filterPaths.addAll(noStarPaths);
- if (noStarPaths.size() == 1) {
- // Transform "select s1 from root.car.* where s1 > 10" to
- // "select s1 from root.car.* where root.car.*.s1 > 10"
- functionOperator.setSinglePath(noStarPaths.get(0));
- return operator;
- } else {
- // Transform "select s1 from root.car.d1, root.car.d2 where s1 > 10" to
- // "select s1 from root.car.d1, root.car.d2 where root.car.d1.s1 > 10 and root.car.d2.s1 > 10"
- // Note that,
- // two fork tree has to be maintained while removing stars in paths for DnfFilterOptimizer
- // requirement.
- return constructBinaryFilterTreeWithAnd(noStarPaths, operator);
- }
- }
-
- private FilterOperator constructBinaryFilterTreeWithAnd(
- List<PartialPath> noStarPaths, FilterOperator operator) throws LogicalOptimizeException {
- FilterOperator filterBinaryTree = new FilterOperator(FilterType.KW_AND);
- FilterOperator currentNode = filterBinaryTree;
- for (int i = 0; i < noStarPaths.size(); i++) {
- if (i > 0 && i < noStarPaths.size() - 1) {
- FilterOperator newInnerNode = new FilterOperator(FilterType.KW_AND);
- currentNode.addChildOperator(newInnerNode);
- currentNode = newInnerNode;
- }
- try {
- if (operator instanceof InOperator) {
- currentNode.addChildOperator(
- new InOperator(
- operator.getFilterType(),
- noStarPaths.get(i),
- ((InOperator) operator).getNot(),
- ((InOperator) operator).getValues()));
- } else if (operator instanceof LikeOperator) {
- currentNode.addChildOperator(
- new LikeOperator(
- operator.getFilterType(),
- noStarPaths.get(i),
- ((LikeOperator) operator).getValue()));
- } else if (operator instanceof RegexpOperator) {
- currentNode.addChildOperator(
- new RegexpOperator(
- operator.getFilterType(),
- noStarPaths.get(i),
- ((RegexpOperator) operator).getValue()));
- } else {
- currentNode.addChildOperator(
- new BasicFunctionOperator(
- operator.getFilterType(),
- noStarPaths.get(i),
- ((BasicFunctionOperator) operator).getValue()));
- }
- } catch (SQLParserException e) {
- throw new LogicalOptimizeException(e.getMessage());
- }
- }
- return filterBinaryTree;
- }
-
- private List<PartialPath> removeWildcardsInConcatPaths(
- List<PartialPath> originalPaths, boolean isPrefixMatch) throws LogicalOptimizeException {
- HashSet<PartialPath> actualPaths = new HashSet<>();
- try {
- for (PartialPath originalPath : originalPaths) {
- List<MeasurementPath> all =
- IoTDB.schemaProcessor.getMeasurementPathsWithAlias(
- originalPath, 0, 0, isPrefixMatch, false)
- .left;
- if (all.isEmpty()) {
- throw new LogicalOptimizeException(
- String.format("Unknown time series %s in `where clause`", originalPath));
- }
- actualPaths.addAll(all);
- }
- } catch (MetadataException e) {
- throw new LogicalOptimizeException("error when remove star: " + e.getMessage());
- }
- return new ArrayList<>(actualPaths);
+ throw new UnsupportedOperationException("");
}
public static <T> void cartesianProduct(
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java b/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java
deleted file mode 100644
index 243979d911..0000000000
--- a/server/src/main/java/org/apache/iotdb/db/qp/utils/WildcardsRemover.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.qp.utils;
-
-import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.commons.path.MeasurementPath;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.exception.query.LogicalOptimizeException;
-import org.apache.iotdb.db.exception.query.PathNumOverLimitException;
-import org.apache.iotdb.db.mpp.plan.expression.Expression;
-import org.apache.iotdb.db.mpp.plan.expression.ResultColumn;
-import org.apache.iotdb.db.qp.logical.crud.QueryOperator;
-import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
-import org.apache.iotdb.db.service.IoTDB;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/** Removes wildcards (applying memory control and slimit/soffset control) */
-public class WildcardsRemover {
-
- private int soffset = 0;
- private int currentOffset = 0;
- private int currentLimit =
- IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum() + 1;
-
- /** Records the path number that the SchemaProcessor totally returned. */
- private int consumed = 0;
-
- /**
- * Since IoTDB v0.13, all DDL and DML use patternMatch as default. Before IoTDB v0.13, all DDL and
- * DML use prefixMatch.
- */
- private boolean isPrefixMatch;
-
- public WildcardsRemover(QueryOperator queryOperator) {
- isPrefixMatch = queryOperator.isPrefixMatchPath();
- if (queryOperator.getSpecialClauseComponent() != null) {
- soffset = queryOperator.getSpecialClauseComponent().getSeriesOffset();
- currentOffset = soffset;
- final int slimit = queryOperator.getSpecialClauseComponent().getSeriesLimit();
- currentLimit = slimit == 0 ? currentLimit : Math.min(slimit, currentLimit);
- }
- }
-
- private WildcardsRemover(boolean isPrefixMatch) {
- this.isPrefixMatch = isPrefixMatch;
- }
-
- public List<MeasurementPath> removeWildcardFrom(PartialPath path)
- throws LogicalOptimizeException {
- try {
- Pair<List<MeasurementPath>, Integer> pair =
- IoTDB.schemaProcessor.getMeasurementPathsWithAlias(
- path, currentLimit, currentOffset, isPrefixMatch, false);
- consumed += pair.right;
- currentOffset -= Math.min(currentOffset, pair.right);
- currentLimit -= pair.left.size();
- return pair.left;
- } catch (MetadataException e) {
- throw new LogicalOptimizeException("error occurred when removing star: " + e.getMessage());
- }
- }
-
- public List<List<Expression>> removeWildcardsFrom(List<Expression> expressions)
- throws LogicalOptimizeException {
- // One by one, remove the wildcards from the input expressions. In most cases, an expression
- // will produce multiple expressions after removing the wildcards. We use extendedExpressions to
- // collect the produced expressions.
- List<List<Expression>> extendedExpressions = new ArrayList<>();
- for (Expression originExpression : expressions) {
- List<Expression> actualExpressions = new ArrayList<>();
- originExpression.removeWildcards(new WildcardsRemover(isPrefixMatch), actualExpressions);
- if (actualExpressions.isEmpty()) {
- // Let's ignore the eval of the function which has at least one non-existence series as
- // input. See IOTDB-1212: https://github.com/apache/iotdb/pull/3101
- return Collections.emptyList();
- }
- extendedExpressions.add(actualExpressions);
- }
-
- // Calculate the Cartesian product of extendedExpressions to get the actual expressions after
- // removing all wildcards. We use actualExpressions to collect them.
- List<List<Expression>> actualExpressions = new ArrayList<>();
- ConcatPathOptimizer.cartesianProduct(
- extendedExpressions, actualExpressions, 0, new ArrayList<>());
-
- // Apply the soffset & slimit control to the actualExpressions and return the remaining
- // expressions.
- List<List<Expression>> remainingExpressions = new ArrayList<>();
- for (List<Expression> actualExpression : actualExpressions) {
- if (currentOffset != 0) {
- --currentOffset;
- continue;
- } else if (currentLimit != 0) {
- --currentLimit;
- } else {
- break;
- }
- remainingExpressions.add(actualExpression);
- }
- consumed += actualExpressions.size();
- return remainingExpressions;
- }
-
- /**
- * Check whether the path number is over limit.
- *
- * @return should break the loop or not
- */
- public boolean checkIfPathNumberIsOverLimit(List<ResultColumn> resultColumns)
- throws PathNumOverLimitException {
- if (resultColumns.size()
- > IoTDBDescriptor.getInstance().getConfig().getMaxQueryDeduplicatedPathNum()) {
- throw new PathNumOverLimitException();
- }
- return currentLimit == 0;
- }
-
- public void checkIfSoffsetIsExceeded(List<ResultColumn> resultColumns)
- throws LogicalOptimizeException {
- if (consumed == 0 ? soffset != 0 : resultColumns.isEmpty()) {
- throw new LogicalOptimizeException(
- String.format(
- "The value of SOFFSET (%d) is equal to or exceeds the number of sequences (%d) that can actually be returned.",
- soffset, consumed));
- }
- }
-}
[iotdb] 05/07: remove some time-consuming operation
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 63e1a212d526e5cd83a4ebc0846c79fe81df2c75
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Dec 5 22:50:22 2022 +0800
remove some time-consuming operation
---
.../iotdb/db/mpp/common/header/DatasetHeader.java | 74 ++++++++++++----------
.../db/mpp/plan/analyze/ConcatPathRewriter.java | 24 +++----
.../mpp/plan/analyze/ExpressionTypeAnalyzer.java | 7 --
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 14 ++--
4 files changed, 58 insertions(+), 61 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java
index 23768dd4c3..4e9a58b262 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.mpp.common.header;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
import com.google.common.primitives.Bytes;
import java.util.ArrayList;
@@ -29,7 +27,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.stream.Collectors;
/** The header of query result dataset. */
public class DatasetHeader {
@@ -45,15 +42,18 @@ public class DatasetHeader {
// map from output column to output tsBlock index
private Map<String, Integer> columnToTsBlockIndexMap;
+ // cached field for create response
+ private List<String> respColumns;
+ private List<String> respDataTypeList;
+ private List<Byte> respAliasColumns;
+ private Map<String, Integer> columnNameIndexMap;
+ private Integer outputValueColumnCount;
+
public DatasetHeader(List<ColumnHeader> columnHeaders, boolean isIgnoreTimestamp) {
this.columnHeaders = columnHeaders;
this.isIgnoreTimestamp = isIgnoreTimestamp;
}
- public List<ColumnHeader> getColumnHeaders() {
- return columnHeaders;
- }
-
public boolean isIgnoreTimestamp() {
return isIgnoreTimestamp;
}
@@ -66,34 +66,36 @@ public class DatasetHeader {
}
public List<String> getRespColumns() {
- return columnHeaders.stream()
- .map(ColumnHeader::getColumnNameWithAlias)
- .collect(Collectors.toList());
- }
-
- public List<String> getColumnNameWithoutAlias() {
- return columnHeaders.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList());
+ if (respColumns == null) {
+ respColumns = new ArrayList<>();
+ for (ColumnHeader columnHeader : columnHeaders) {
+ respColumns.add(columnHeader.getColumnNameWithAlias());
+ }
+ }
+ return respColumns;
}
public List<String> getRespDataTypeList() {
- return columnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .map(Objects::toString)
- .collect(Collectors.toList());
- }
-
- public List<TSDataType> getRespDataTypes() {
- return columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList());
+ if (respDataTypeList == null) {
+ respDataTypeList = new ArrayList<>();
+ for (ColumnHeader columnHeader : columnHeaders) {
+ respDataTypeList.add(columnHeader.getColumnType().toString());
+ }
+ }
+ return respDataTypeList;
}
public List<Byte> getRespAliasColumns() {
- BitSet aliasMap = new BitSet();
- for (int i = 0; i < columnHeaders.size(); ++i) {
- if (columnHeaders.get(i).hasAlias()) {
- aliasMap.set(i);
+ if (respAliasColumns == null) {
+ BitSet aliasMap = new BitSet();
+ for (int i = 0; i < columnHeaders.size(); ++i) {
+ if (columnHeaders.get(i).hasAlias()) {
+ aliasMap.set(i);
+ }
}
+ respAliasColumns = new ArrayList<>(Bytes.asList(aliasMap.toByteArray()));
}
- return new ArrayList<>(Bytes.asList(aliasMap.toByteArray()));
+ return respAliasColumns;
}
public Map<String, Integer> getColumnNameIndexMap() {
@@ -101,17 +103,23 @@ public class DatasetHeader {
return columnToTsBlockIndexMap;
}
- Map<String, Integer> columnNameIndexMap = new HashMap<>();
- for (ColumnHeader columnHeader : columnHeaders) {
- columnNameIndexMap.put(
- columnHeader.getColumnNameWithAlias(),
- columnToTsBlockIndexMap.get(columnHeader.getColumnName()));
+ if (columnNameIndexMap == null) {
+ columnNameIndexMap = new HashMap<>();
+ for (ColumnHeader columnHeader : columnHeaders) {
+ columnNameIndexMap.put(
+ columnHeader.getColumnNameWithAlias(),
+ columnToTsBlockIndexMap.get(columnHeader.getColumnName()));
+ }
}
return columnNameIndexMap;
}
public int getOutputValueColumnCount() {
- return (int) columnHeaders.stream().map(ColumnHeader::getColumnName).distinct().count();
+ if (outputValueColumnCount == null) {
+ outputValueColumnCount =
+ (int) columnHeaders.stream().map(ColumnHeader::getColumnName).distinct().count();
+ }
+ return outputValueColumnCount;
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ConcatPathRewriter.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ConcatPathRewriter.java
index 71273d5c12..5db7080d53 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ConcatPathRewriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ConcatPathRewriter.java
@@ -30,7 +30,6 @@ import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement;
import java.util.ArrayList;
import java.util.List;
-import java.util.stream.Collectors;
/**
* This rewriter:
@@ -53,12 +52,10 @@ public class ConcatPathRewriter {
List<PartialPath> prefixPaths = queryStatement.getFromComponent().getPrefixPaths();
if (queryStatement.isAlignByDevice()) {
- queryStatement.getSelectComponent().getResultColumns().stream()
- .map(ResultColumn::getExpression)
- .forEach(
- expression ->
- ExpressionAnalyzer.constructPatternTreeFromExpression(
- expression, prefixPaths, patternTree));
+ for (ResultColumn resultColumn : queryStatement.getSelectComponent().getResultColumns()) {
+ ExpressionAnalyzer.constructPatternTreeFromExpression(
+ resultColumn.getExpression(), prefixPaths, patternTree);
+ }
} else {
// concat SELECT with FROM
List<ResultColumn> resultColumns =
@@ -101,13 +98,12 @@ public class ConcatPathRewriter {
String.format(
"alias '%s' can only be matched with one time series", resultColumn.getAlias()));
}
- resultColumns.addAll(
- resultExpressions.stream()
- .map(
- expression ->
- new ResultColumn(
- expression, resultColumn.getAlias(), resultColumn.getColumnType()))
- .collect(Collectors.toList()));
+
+ for (Expression resultExpression : resultExpressions) {
+ resultColumns.add(
+ new ResultColumn(
+ resultExpression, resultColumn.getAlias(), resultColumn.getColumnType()));
+ }
}
return resultColumns;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionTypeAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionTypeAnalyzer.java
index 83f61afd46..145b8e92b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionTypeAnalyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionTypeAnalyzer.java
@@ -48,8 +48,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static com.google.common.base.Preconditions.checkArgument;
-
public class ExpressionTypeAnalyzer {
private final Map<NodeRef<Expression>, TSDataType> expressionTypes = new LinkedHashMap<>();
@@ -252,11 +250,6 @@ public class ExpressionTypeAnalyzer {
}
if (functionExpression.isBuiltInAggregationFunctionExpression()) {
- checkArgument(
- inputExpressions.size() == 1,
- String.format(
- "Builtin aggregation function only accepts 1 input expression. Actual %d input expressions.",
- inputExpressions.size()));
return setExpressionType(
functionExpression,
TypeInferenceUtils.getAggrDataType(
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
index 3485e62eaa..477c26575f 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java
@@ -29,19 +29,19 @@ import org.apache.commons.lang3.StringUtils;
public class TypeInferenceUtils {
- private static TSDataType booleanStringInferType =
+ private static final TSDataType booleanStringInferType =
IoTDBDescriptor.getInstance().getConfig().getBooleanStringInferType();
- private static TSDataType integerStringInferType =
+ private static final TSDataType integerStringInferType =
IoTDBDescriptor.getInstance().getConfig().getIntegerStringInferType();
- private static TSDataType longStringInferType =
+ private static final TSDataType longStringInferType =
IoTDBDescriptor.getInstance().getConfig().getLongStringInferType();
- private static TSDataType floatingStringInferType =
+ private static final TSDataType floatingStringInferType =
IoTDBDescriptor.getInstance().getConfig().getFloatingStringInferType();
- private static TSDataType nanStringInferType =
+ private static final TSDataType nanStringInferType =
IoTDBDescriptor.getInstance().getConfig().getNanStringInferType();
private TypeInferenceUtils() {}
@@ -119,7 +119,7 @@ public class TypeInferenceUtils {
"Aggregate functions [AVG, SUM, EXTREME, MIN_VALUE, MAX_VALUE] only support numeric data types [INT32, INT64, FLOAT, DOUBLE]");
}
- switch (aggrFuncName.toLowerCase()) {
+ switch (aggrFuncName) {
case SQLConstant.MIN_TIME:
case SQLConstant.MAX_TIME:
case SQLConstant.COUNT:
@@ -140,7 +140,7 @@ public class TypeInferenceUtils {
private static boolean verifyIsAggregationDataTypeMatched(
String aggrFuncName, TSDataType dataType) {
- switch (aggrFuncName.toLowerCase()) {
+ switch (aggrFuncName) {
case SQLConstant.AVG:
case SQLConstant.SUM:
case SQLConstant.EXTREME:
[iotdb] 07/07: make code more efficient
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/OptSingleGroupby
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6f76c25c0b4956ff2898247d2d79a72925780eb3
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Dec 6 09:36:55 2022 +0800
make code more efficient
---
.../apache/iotdb/commons/path/PathPatternTree.java | 4 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 4 +-
.../db/engine/storagegroup/TsFileResource.java | 20 +++++-----
.../iotdb/db/mpp/common/FragmentInstanceId.java | 2 +-
.../iotdb/db/mpp/common/header/DatasetHeader.java | 8 +++-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 17 +++++----
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 20 ++++++----
.../plan/parameter/AggregationDescriptor.java | 17 +++++----
.../query/control/clientsession/ClientSession.java | 2 +-
.../java/org/apache/iotdb/db/utils/QueryUtils.java | 39 +++++++++----------
.../org/apache/iotdb/db/utils/SetThreadName.java | 5 +--
.../tsfile/read/reader/page/AlignedPageReader.java | 44 ++++++++++++++++++++++
12 files changed, 116 insertions(+), 66 deletions(-)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java b/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
index d06bbe7106..45ce4558f7 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
@@ -35,7 +35,6 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
public class PathPatternTree {
@@ -87,8 +86,7 @@ public class PathPatternTree {
}
if (!isExist) {
// remove duplicate path in pathPatternList
- pathPatternList.removeAll(
- pathPatternList.stream().filter(pathPattern::include).collect(Collectors.toList()));
+ pathPatternList.removeIf(p -> p.include(pathPattern));
pathPatternList.add(pathPattern);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index c32c4f8f59..e2c6a29346 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -1788,7 +1788,7 @@ public class DataRegion {
// for upgrade files and old files must be closed
for (TsFileResource tsFileResource : upgradeTsFileResources) {
- if (!tsFileResource.isSatisfied(
+ if (tsFileResource.unsatisfied(
singleDeviceId, timeFilter, isSeq, dataTTL, context.isDebug())) {
continue;
}
@@ -1801,7 +1801,7 @@ public class DataRegion {
}
for (TsFileResource tsFileResource : tsFileResources) {
- if (!tsFileResource.isSatisfied(
+ if (tsFileResource.unsatisfied(
singleDeviceId, timeFilter, isSeq, dataTTL, context.isDebug())) {
continue;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index a73ada0e33..0049b3bf47 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -639,10 +639,10 @@ public class TsFileResource {
}
/** @return true if the device is contained in the TsFile and it lives beyond TTL */
- public boolean isSatisfied(
+ public boolean unsatisfied(
String deviceId, Filter timeFilter, boolean isSeq, long ttl, boolean debug) {
if (deviceId == null) {
- return isSatisfied(timeFilter, isSeq, ttl, debug);
+ return !isSatisfied(timeFilter, isSeq, ttl, debug);
}
long[] startAndEndTime = timeIndex.getStartAndEndTime(deviceId);
@@ -653,17 +653,17 @@ public class TsFileResource {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
}
- return false;
+ return true;
}
long startTime = startAndEndTime[0];
long endTime = isClosed() || !isSeq ? startAndEndTime[1] : Long.MAX_VALUE;
- if (!isAlive(endTime, ttl)) {
+ if (notAlive(endTime, ttl)) {
if (debug) {
DEBUG_LOGGER.info("file {} is not satisfied because of ttl!", file);
}
- return false;
+ return true;
}
if (timeFilter != null) {
@@ -672,9 +672,9 @@ public class TsFileResource {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of time filter!", deviceId, fsFactory);
}
- return res;
+ return !res;
}
- return true;
+ return false;
}
/** @return true if the TsFile lives beyond TTL */
@@ -682,7 +682,7 @@ public class TsFileResource {
long startTime = getFileStartTime();
long endTime = isClosed() || !isSeq ? getFileEndTime() : Long.MAX_VALUE;
- if (!isAlive(endTime, ttl)) {
+ if (notAlive(endTime, ttl)) {
if (debug) {
DEBUG_LOGGER.info("file {} is not satisfied because of ttl!", file);
}
@@ -733,8 +733,8 @@ public class TsFileResource {
}
/** @return whether the given time falls in ttl */
- private boolean isAlive(long time, long dataTTL) {
- return dataTTL == Long.MAX_VALUE || (DateTimeUtils.currentTime() - time) <= dataTTL;
+ private boolean notAlive(long time, long dataTTL) {
+ return dataTTL != Long.MAX_VALUE && (DateTimeUtils.currentTime() - time) > dataTTL;
}
public void setProcessor(TsFileProcessor processor) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/FragmentInstanceId.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/FragmentInstanceId.java
index e75d264700..b1c12bb729 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/FragmentInstanceId.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/FragmentInstanceId.java
@@ -107,6 +107,6 @@ public class FragmentInstanceId {
}
public static String createFullId(String queryId, int fragmentId, String instanceId) {
- return String.format("%s.%d.%s", queryId, fragmentId, instanceId);
+ return queryId + '.' + fragmentId + '.' + instanceId;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java
index 4e9a58b262..c835970c17 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/DatasetHeader.java
@@ -24,6 +24,7 @@ import com.google.common.primitives.Bytes;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -116,8 +117,11 @@ public class DatasetHeader {
public int getOutputValueColumnCount() {
if (outputValueColumnCount == null) {
- outputValueColumnCount =
- (int) columnHeaders.stream().map(ColumnHeader::getColumnName).distinct().count();
+ HashSet<String> columnNameSet = new HashSet<>();
+ for (ColumnHeader columnHeader : columnHeaders) {
+ columnNameSet.add(columnHeader.getColumnName());
+ }
+ outputValueColumnCount = columnNameSet.size();
}
return outputValueColumnCount;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
index 340a8f4bcc..4e76b768d5 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
@@ -34,11 +34,11 @@ import com.google.common.util.concurrent.SettableFuture;
import javax.annotation.concurrent.NotThreadSafe;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* One dataDriver is responsible for one FragmentInstance which is for data query, which may
@@ -125,17 +125,20 @@ public class DataDriver extends Driver {
DataRegion dataRegion = context.getDataRegion();
dataRegion.readLock();
try {
- List<PartialPath> pathList =
- context.getPaths().stream().map(IDTable::translateQueryPath).collect(Collectors.toList());
- // when all the selected series are under the same device, the QueryDataSource will be
- // filtered according to timeIndex
- Set<String> selectedDeviceIdSet =
- pathList.stream().map(PartialPath::getDevice).collect(Collectors.toSet());
+ List<PartialPath> pathList = new ArrayList<>();
+ Set<String> selectedDeviceIdSet = new HashSet<>();
+ for (PartialPath path : context.getPaths()) {
+ PartialPath translatedPath = IDTable.translateQueryPath(path);
+ pathList.add(translatedPath);
+ selectedDeviceIdSet.add(translatedPath.getDevice());
+ }
Filter timeFilter = context.getTimeFilter();
QueryDataSource dataSource =
dataRegion.query(
pathList,
+ // when all the selected series are under the same device, the QueryDataSource will be
+ // filtered according to timeIndex
selectedDeviceIdSet.size() == 1 ? selectedDeviceIdSet.iterator().next() : null,
driverContext.getFragmentInstanceContext(),
timeFilter != null ? timeFilter.copy() : null);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
index a08f9af268..5b86b81d52 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
@@ -133,10 +133,12 @@ public class LocalExecutionPlanner {
TSStatusCode.MPP_MEMORY_NOT_ENOUGH.getStatusCode());
} else {
freeMemoryForOperators -= estimatedMemorySize;
- LOGGER.debug(
- String.format(
- "[ConsumeMemory] consume: %d, current remaining memory: %d",
- estimatedMemorySize, freeMemoryForOperators));
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "[ConsumeMemory] consume: {}, current remaining memory: {}",
+ estimatedMemorySize,
+ freeMemoryForOperators);
+ }
}
}
@@ -147,10 +149,12 @@ public class LocalExecutionPlanner {
new SetThreadName(stateMachine.getFragmentInstanceId().getFullId())) {
synchronized (this) {
this.freeMemoryForOperators += estimatedMemorySize;
- LOGGER.debug(
- String.format(
- "[ReleaseMemory] release: %d, current remaining memory: %d",
- estimatedMemorySize, freeMemoryForOperators));
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "[ReleaseMemory] release: {}, current remaining memory: {}",
+ estimatedMemorySize,
+ freeMemoryForOperators);
+ }
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/AggregationDescriptor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/AggregationDescriptor.java
index 464ee03130..ec523e38ff 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/AggregationDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/AggregationDescriptor.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.mpp.plan.planner.plan.parameter;
import org.apache.iotdb.db.mpp.plan.expression.Expression;
+import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.query.aggregation.AggregationType;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -125,22 +126,22 @@ public class AggregationDescriptor {
if (isPartial) {
switch (aggregationType) {
case AVG:
- outputAggregationNames.add(AggregationType.COUNT.name().toLowerCase());
- outputAggregationNames.add(AggregationType.SUM.name().toLowerCase());
+ outputAggregationNames.add(SQLConstant.COUNT);
+ outputAggregationNames.add(SQLConstant.SUM);
break;
case FIRST_VALUE:
- outputAggregationNames.add(AggregationType.FIRST_VALUE.name().toLowerCase());
- outputAggregationNames.add(AggregationType.MIN_TIME.name().toLowerCase());
+ outputAggregationNames.add(SQLConstant.FIRST_VALUE);
+ outputAggregationNames.add(SQLConstant.MIN_TIME);
break;
case LAST_VALUE:
- outputAggregationNames.add(AggregationType.LAST_VALUE.name().toLowerCase());
- outputAggregationNames.add(AggregationType.MAX_TIME.name().toLowerCase());
+ outputAggregationNames.add(SQLConstant.LAST_VALUE);
+ outputAggregationNames.add(SQLConstant.MAX_TIME);
break;
default:
- outputAggregationNames.add(aggregationFuncName.toLowerCase());
+ outputAggregationNames.add(aggregationFuncName);
}
} else {
- outputAggregationNames.add(aggregationFuncName.toLowerCase());
+ outputAggregationNames.add(aggregationFuncName);
}
return outputAggregationNames;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
index 68150b2837..77b0174c3a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
@@ -55,7 +55,7 @@ public class ClientSession extends IClientSession {
@Override
String getConnectionId() {
- return String.format("%s:%s", getClientAddress(), getClientPort());
+ return getClientAddress() + ':' + getClientPort();
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 3020ee54e4..b7e27b3098 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -29,10 +29,9 @@ import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
+import java.util.Comparator;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
+import java.util.TreeMap;
public class QueryUtils {
@@ -194,26 +193,24 @@ public class QueryUtils {
}
}
+ private static final Comparator<Long> descendingComparator = (o1, o2) -> Long.compare(o2, o1);
+
public static void fillOrderIndexes(
QueryDataSource dataSource, String deviceId, boolean ascending) {
List<TsFileResource> unseqResources = dataSource.getUnseqResources();
- int[] orderIndex = new int[unseqResources.size()];
- AtomicInteger index = new AtomicInteger();
- Map<Integer, Long> intToOrderTimeMap =
- unseqResources.stream()
- .collect(
- Collectors.toMap(
- key -> index.getAndIncrement(),
- resource -> resource.getOrderTime(deviceId, ascending)));
- index.set(0);
- intToOrderTimeMap.entrySet().stream()
- .sorted(
- (t1, t2) ->
- ascending
- ? Long.compare(t1.getValue(), t2.getValue())
- : Long.compare(t2.getValue(), t1.getValue()))
- .collect(Collectors.toList())
- .forEach(item -> orderIndex[index.getAndIncrement()] = item.getKey());
- dataSource.setUnSeqFileOrderIndex(orderIndex);
+
+ TreeMap<Long, Integer> intToOrderTimeMapV2 =
+ ascending ? new TreeMap<>() : new TreeMap<>(descendingComparator);
+ int index = 0;
+ for (TsFileResource resource : unseqResources) {
+ intToOrderTimeMapV2.put(resource.getOrderTime(deviceId, ascending), index++);
+ }
+
+ index = 0;
+ int[] orderIndexes = new int[unseqResources.size()];
+ for (Integer orderIndex : intToOrderTimeMapV2.values()) {
+ orderIndexes[index++] = orderIndex;
+ }
+ dataSource.setUnSeqFileOrderIndex(orderIndexes);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SetThreadName.java b/server/src/main/java/org/apache/iotdb/db/utils/SetThreadName.java
index 3dca499262..43adca20b2 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SetThreadName.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SetThreadName.java
@@ -30,10 +30,9 @@ public class SetThreadName implements Closeable {
originalThreadName = Thread.currentThread().getName();
int index = originalThreadName.indexOf("$");
if (index < 0) {
- Thread.currentThread().setName(String.format("%s$%s", originalThreadName, suffix));
+ Thread.currentThread().setName(originalThreadName + '$' + suffix);
} else {
- Thread.currentThread()
- .setName(String.format("%s$%s", originalThreadName.substring(0, index), suffix));
+ Thread.currentThread().setName(originalThreadName.substring(0, index) + '$' + suffix);
}
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
index 3f076906bb..ffbe380131 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
@@ -115,6 +115,10 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
@Override
public TsBlock getAllSatisfiedData() throws IOException {
+ if (!isModified()) {
+ return getAllSatisfiedDataWithoutModified();
+ }
+
builder.reset();
long[] timeBatch = timePageReader.getNextTimeBatch();
@@ -192,6 +196,46 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader {
return builder.build();
}
+ public TsBlock getAllSatisfiedDataWithoutModified() throws IOException {
+ builder.reset();
+ long[] timeBatch = timePageReader.getNextTimeBatch();
+
+ // if all the sub sensors' value are null in current row, just discard it
+ // if !filter.satisfy, discard this row
+ boolean[] keepCurrentRow = new boolean[timeBatch.length];
+ if (filter == null) {
+ Arrays.fill(keepCurrentRow, true);
+ } else {
+ for (int i = 0, n = timeBatch.length; i < n; i++) {
+ keepCurrentRow[i] = filter.satisfy(timeBatch[i], null);
+ }
+ }
+
+ // construct time column
+ for (int i = 0; i < timeBatch.length; i++) {
+ if (keepCurrentRow[i]) {
+ builder.getTimeColumnBuilder().writeLong(timeBatch[i]);
+ builder.declarePosition();
+ }
+ }
+
+ // construct value columns
+ for (int i = 0; i < valueCount; i++) {
+ ValuePageReader pageReader = valuePageReaderList.get(i);
+ if (pageReader != null) {
+ pageReader.writeColumnBuilderWithNextBatch(
+ timeBatch, builder.getColumnBuilder(i), keepCurrentRow);
+ } else {
+ for (int j = 0; j < timeBatch.length; j++) {
+ if (keepCurrentRow[j]) {
+ builder.getColumnBuilder(i).appendNull();
+ }
+ }
+ }
+ }
+ return builder.build();
+ }
+
public void setDeleteIntervalList(List<List<TimeRange>> list) {
for (int i = 0; i < valueCount; i++) {
if (valuePageReaderList.get(i) != null) {