You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/03/01 11:07:14 UTC
(doris) branch branch-2.1 updated (493c9d49eab -> 874f4c693b2)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from 493c9d49eab [fix](Nereids) check slot from children may produce wrong format err (#31638)
new b26dcf26774 [opt](nereids) Composite predicate supports range predicate when rewritting by materialzied view (#31538)
new 248ea209014 Revert "[test](regression) add regression test for schange change of complex …" (#31660)
new 38cb17567a5 [opt](plsql) Fix plsql exception and doris exception compatibility (#31647)
new 874f4c693b2 [feat](nereids) support nullSafeEqual estimation (#31616)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hive/scripts/create_preinstalled_table.hql | 14 ---
.../mv/AbstractMaterializedViewRule.java | 3 +-
.../exploration/mv/MaterializedViewUtils.java | 3 +
.../nereids/rules/exploration/mv/Predicates.java | 44 ++++---
.../rules/exploration/mv/PredicatesSplitter.java | 12 ++
.../rules/expression/rules/SimplifyRange.java | 9 +-
.../doris/nereids/stats/FilterEstimation.java | 15 ++-
.../apache/doris/nereids/util/ExpressionUtils.java | 4 +
.../main/java/org/apache/doris/plsql/Console.java | 8 +-
.../src/main/java/org/apache/doris/plsql/Exec.java | 44 ++++---
.../src/main/java/org/apache/doris/plsql/Stmt.java | 8 +-
.../doris/plsql/executor/PlSqlOperation.java | 46 +++----
.../apache/doris/plsql/executor/PlsqlResult.java | 35 +++---
.../org/apache/doris/qe/MysqlConnectProcessor.java | 2 +-
.../org/apache/doris/statistics/Statistics.java | 6 +-
.../rules/expression/SimplifyRangeTest.java | 134 +++++++++++++++++++++
.../doris/nereids/stats/FilterEstimationTest.java | 36 ++++++
.../hive/test_hive_parquet.groovy | 10 --
.../filter_equal_or_notequal.groovy | 104 ++++++++++------
19 files changed, 391 insertions(+), 146 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/04: Revert "[test](regression) add regression test for schange change of complex …" (#31660)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 248ea2090145077267642b11bca7cc74c59d9a39
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Fri Mar 1 18:35:01 2024 +0800
Revert "[test](regression) add regression test for schange change of complex …" (#31660)
This reverts commit dcd2afdb4e857791fed66a46f28ab3adc25494e1.
Reverts #31207
---
.../hive/scripts/create_preinstalled_table.hql | 14 --------------
.../suites/external_table_p0/hive/test_hive_parquet.groovy | 10 ----------
2 files changed, 24 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
index 00d2a699410..9baf13020f6 100644
--- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
+++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql
@@ -1937,17 +1937,3 @@ with serdeproperties
,'seperatorChar'=',');
insert into employee_gz values ('a', '1.1'), ('b', '2.2');
-
-create database hive_schema_change;
-use hive_schema_change;
-
-create table struct_test (
- id int,
- sf struct<f1: int, f2: map<string, string>>) stored as parquet;
-
-insert into struct_test values
-(1, named_struct('f1', 1, 'f2', str_to_map('1:s2,2:s2'))),
-(2, named_struct('f1', 2, 'f2', str_to_map('k1:s3,k2:s4'))),
-(3, named_struct('f1', 3, 'f2', str_to_map('k1:s5,k2:s6')));
-
-alter table struct_test change sf sf struct<f1:int, f2: string>;
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy b/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
index dff86e77285..c381a3c236b 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
@@ -169,15 +169,6 @@ suite("test_hive_parquet", "p0,external,hive,external_docker,external_docker_hiv
"""
}
- def q23 = {
- sql """use hive_schema_change"""
- // not support the schema change of complex type
- test {
- sql "select * from struct_test"
- exception "Wrong data type for column 'f2'"
- }
- }
-
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
try {
@@ -216,7 +207,6 @@ suite("test_hive_parquet", "p0,external,hive,external_docker,external_docker_hiv
q20()
q21()
q22()
- q23()
sql """explain physical plan select l_partkey from partition_table
where (nation != 'cn' or city !='beijing') and (l_quantity > 28 or l_extendedprice > 30000)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 03/04: [opt](plsql) Fix plsql exception and doris exception compatibility (#31647)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 38cb17567a575345329d10ca49807d59fe94d130
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Fri Mar 1 18:44:23 2024 +0800
[opt](plsql) Fix plsql exception and doris exception compatibility (#31647)
---
.../main/java/org/apache/doris/plsql/Console.java | 8 +++-
.../src/main/java/org/apache/doris/plsql/Exec.java | 44 +++++++++++++--------
.../src/main/java/org/apache/doris/plsql/Stmt.java | 8 ++--
.../doris/plsql/executor/PlSqlOperation.java | 46 ++++++++++++----------
.../apache/doris/plsql/executor/PlsqlResult.java | 35 +++++++++-------
.../org/apache/doris/qe/MysqlConnectProcessor.java | 2 +-
6 files changed, 84 insertions(+), 59 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/Console.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/Console.java
index 024ea356759..91cc0988652 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/Console.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/Console.java
@@ -20,6 +20,8 @@
package org.apache.doris.plsql;
+import org.apache.doris.common.ErrorCode;
+
public interface Console {
void print(String msg);
@@ -27,7 +29,7 @@ public interface Console {
void printError(String msg);
- void flushConsole();
+ void printError(ErrorCode errorCode, String msg);
Console STANDARD = new Console() {
@Override
@@ -46,6 +48,8 @@ public interface Console {
}
@Override
- public void flushConsole() {}
+ public void printError(ErrorCode errorCode, String msg) {
+ System.err.println(errorCode.toString() + ", " + msg);
+ }
};
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java
index 34e84ab4097..3caf8181ff0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/Exec.java
@@ -20,6 +20,7 @@
package org.apache.doris.plsql;
+import org.apache.doris.common.ErrorCode;
import org.apache.doris.nereids.PLLexer;
import org.apache.doris.nereids.PLParser;
import org.apache.doris.nereids.PLParser.Allocate_cursor_stmtContext;
@@ -150,6 +151,8 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNode;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import java.io.ByteArrayInputStream;
import java.io.Closeable;
@@ -176,6 +179,7 @@ import java.util.stream.Collectors;
* PL/SQL script executor
*/
public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer> implements Closeable {
+ private static final Logger LOG = LogManager.getLogger(Exec.class);
public static final String VERSION = "PL/SQL 0.1";
public static final String ERRORCODE = "ERRORCODE";
@@ -839,6 +843,8 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
init();
try {
parseAndEval(arguments);
+ } catch (Exception e) {
+ exec.signal(e);
} finally {
close();
}
@@ -901,8 +907,6 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
*/
public void init() {
enterGlobalScope();
- // specify the default log4j2 properties file.
- System.setProperty("log4j.configurationFile", "hive-log4j2.properties");
if (conf == null) {
conf = new Conf();
}
@@ -944,7 +948,7 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
PLParser parser = newParser(tokens);
ParseTree tree = parser.program();
if (trace) {
- console.printError("Parser tree: " + tree.toStringTree(parser));
+ console.printLine("Parser tree: " + tree.toStringTree(parser));
}
return tree;
}
@@ -978,7 +982,7 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
boolean parseArguments(String[] args) {
boolean parsed = arguments.parse(args);
if (parsed && arguments.hasVersionOption()) {
- console.printError(VERSION);
+ console.printLine(VERSION);
return false;
}
if (!parsed || arguments.hasHelpOption()
@@ -1092,13 +1096,24 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
if (sig.type == Signal.Type.VALIDATION) {
error(((PlValidationException) sig.exception).getCtx(), sig.exception.getMessage());
} else if (sig.type == Signal.Type.SQLEXCEPTION) {
- console.printError("Unhandled exception in PL/SQL. " + ExceptionUtils.getStackTrace(sig.exception));
+ LOG.warn(ExceptionUtils.getStackTrace(sig.exception));
+ console.printError(ErrorCode.ERR_SP_BAD_SQLSTATE,
+ "Unhandled exception in PL/SQL. " + sig.exception.toString());
} else if (sig.type == Signal.Type.UNSUPPORTED_OPERATION) {
- console.printError(sig.value == null ? "Unsupported operation" : sig.value);
+ console.printError(ErrorCode.ERR_SP_BAD_SQLSTATE,
+ sig.value == null ? "Unsupported operation" : sig.value);
+ } else if (sig.type == Signal.Type.TOO_MANY_ROWS) {
+ console.printError(ErrorCode.ERR_SP_BAD_SQLSTATE,
+ sig.value == null ? "Too many rows exception" : sig.value);
+ } else if (sig.type == Signal.Type.NOTFOUND) {
+ console.printError(ErrorCode.ERR_SP_FETCH_NO_DATA,
+ sig.value == null ? "Not found data exception" : sig.value);
} else if (sig.exception != null) {
- console.printError("HPL/SQL error: " + ExceptionUtils.getStackTrace(sig.exception));
+ LOG.warn(ExceptionUtils.getStackTrace(sig.exception));
+ console.printError(ErrorCode.ERR_SP_BAD_SQLSTATE,
+ "PL/SQL error: " + sig.exception.toString());
} else if (sig.value != null) {
- console.printError(sig.value);
+ console.printError(ErrorCode.ERR_SP_BAD_SQLSTATE, sig.value);
} else {
trace(null, "Signal: " + sig.type);
}
@@ -1156,14 +1171,9 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
@Override
public Integer visitDoris_statement(Doris_statementContext ctx) {
Integer rc = exec.stmt.statement(ctx);
- if (rc != 0) {
- printExceptions();
- throw new RuntimeException(exec.signalPeek().getValue());
- }
// Sometimes the query results are not returned to the mysql client,
- // such as declare result; select … into result;
+ // such as `declare result; select … into result;`, not need finalize.
resultListener.onFinalize();
- console.flushConsole(); // if running from plsql.sh
return rc;
}
@@ -1793,6 +1803,8 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
} else if (ctx.multipartIdentifier() != null) {
functionCall(ctx, ctx.multipartIdentifier(), null);
}
+ } catch (Exception e) {
+ exec.signal(e);
} finally {
exec.inCallStmt = false;
}
@@ -2379,9 +2391,9 @@ public class Exec extends org.apache.doris.nereids.PLParserBaseVisitor<Integer>
return;
}
if (ctx != null) {
- console.printError("Ln:" + ctx.getStart().getLine() + " " + message);
+ console.printLine("Ln:" + ctx.getStart().getLine() + " " + message);
} else {
- console.printError(message);
+ console.printLine(message);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/Stmt.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/Stmt.java
index 1567f020701..d86a648b99a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/Stmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/Stmt.java
@@ -134,11 +134,11 @@ public class Stmt {
exec.setSqlSuccess();
if (query.next()) {
exec.setSqlCode(SqlCodes.TOO_MANY_ROWS);
- exec.signal(Signal.Type.TOO_MANY_ROWS);
+ exec.signal(Signal.Type.TOO_MANY_ROWS, "too many rows into variables");
}
} else {
exec.setSqlCode(SqlCodes.NO_DATA_FOUND);
- exec.signal(Signal.Type.NOTFOUND);
+ exec.signal(Signal.Type.NOTFOUND, "no rows into variables");
}
} else if (ctx instanceof Doris_statementContext) { // only from visitStatement
// Print all results for standalone Statement.
@@ -406,7 +406,7 @@ public class Stmt {
return 1;
} else if (exec.getOffline()) {
exec.setSqlCode(SqlCodes.NO_DATA_FOUND);
- exec.signal(Signal.Type.NOTFOUND);
+ exec.signal(Signal.Type.NOTFOUND, "fetch not found data");
return 0;
}
// Assign values from the row to local variables
@@ -590,7 +590,7 @@ public class Stmt {
exec.setSqlSuccess();
} else {
exec.setSqlCode(SqlCodes.NO_DATA_FOUND);
- exec.signal(Signal.Type.NOTFOUND);
+ exec.signal(Signal.Type.NOTFOUND, "assign from select not found data");
}
} catch (QueryException | AnalysisException e) {
exec.signal(query);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlSqlOperation.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlSqlOperation.java
index b98507c1a41..8c5baf23764 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlSqlOperation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlSqlOperation.java
@@ -44,30 +44,34 @@ public class PlSqlOperation {
}
public void execute(ConnectContext ctx, String statement) {
- ctx.setRunProcedure(true);
- ctx.setProcedureExec(exec);
- result.reset();
try {
- Arguments args = new Arguments();
- args.parse(new String[] { "-e", statement });
- exec.parseAndEval(args);
- // Exception is not thrown after catch.
- // For example, select a not exist table will return empty results, exception
- // will put into signals.
- exec.printExceptions();
- String error = result.getError();
- String msg = result.getMsg();
- if (!error.isEmpty()) {
- ctx.getState().setError("plsql exec error, " + error);
- } else if (!msg.isEmpty()) {
- ctx.getState().setOk(0, 0, msg);
+ ctx.setRunProcedure(true);
+ ctx.setProcedureExec(exec);
+ result.reset();
+ try {
+ Arguments args = new Arguments();
+ args.parse(new String[] {"-e", statement});
+ exec.parseAndEval(args);
+ } catch (Exception e) {
+ exec.signal(e);
+ } finally {
+ // Exception is not thrown after catch.
+ // For example, select a not exist table will return empty results, exception
+ // will put into signals.
+ exec.printExceptions();
+ String error = result.getError();
+ String msg = result.getMsg();
+ if (!error.isEmpty()) {
+ ctx.getState().setError(result.getLastErrorCode(), error);
+ } else if (!msg.isEmpty()) {
+ ctx.getState().setOk(0, 0, msg);
+ }
+ ctx.getMysqlChannel().reset();
+ ctx.setRunProcedure(false);
+ ctx.setProcedureExec(null);
}
- ctx.getMysqlChannel().reset();
- ctx.setRunProcedure(false);
- ctx.setProcedureExec(null);
} catch (Exception e) {
- exec.printExceptions();
- ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, result.getError() + " " + e.getMessage());
+ ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getMessage());
LOG.warn(e);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlResult.java b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlResult.java
index c632b930a8a..76e600e7fcc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plsql/executor/PlsqlResult.java
@@ -17,6 +17,7 @@
package org.apache.doris.plsql.executor;
+import org.apache.doris.common.ErrorCode;
import org.apache.doris.mysql.MysqlEofPacket;
import org.apache.doris.mysql.MysqlSerializer;
import org.apache.doris.mysql.MysqlServerStatusFlag;
@@ -27,11 +28,13 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ConnectProcessor;
import org.apache.doris.qe.QueryState;
+import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.List;
// If running from mysql client, first send schema column,
// and then send the ByteBuffer through the mysql channel.
@@ -44,12 +47,14 @@ public class PlsqlResult implements ResultListener, Console {
private Metadata metadata = null;
private StringBuilder msg;
private StringBuilder error;
+ private List<ErrorCode> errorCodes;
private boolean isSendFields;
public PlsqlResult() {
this.msg = new StringBuilder();
this.error = new StringBuilder();
this.isSendFields = false;
+ this.errorCodes = Lists.newArrayList();
}
public void reset() {
@@ -58,6 +63,7 @@ public class PlsqlResult implements ResultListener, Console {
isSendFields = false;
error.delete(0, error.length());
msg.delete(0, msg.length());
+ errorCodes.clear();
}
public void setProcessor(ConnectProcessor processor) {
@@ -72,6 +78,14 @@ public class PlsqlResult implements ResultListener, Console {
return error.toString();
}
+ public List<ErrorCode> getErrorCodes() {
+ return errorCodes;
+ }
+
+ public ErrorCode getLastErrorCode() {
+ return errorCodes.isEmpty() ? ErrorCode.ERR_UNKNOWN_ERROR : errorCodes.get(errorCodes.size() - 1);
+ }
+
@Override
public void onMysqlRow(ByteBuffer rows) {
ConnectContext ctx = processor != null ? processor.getConnectContext() : ConnectContext.get();
@@ -106,6 +120,7 @@ public class PlsqlResult implements ResultListener, Console {
@Override
public void onFinalize() {
+ // If metadata not null, it means that mysql channel sent query results.
if (metadata == null) {
return;
}
@@ -166,24 +181,14 @@ public class PlsqlResult implements ResultListener, Console {
@Override
public void printError(String msg) {
- this.error.append(msg);
+ errorCodes.add(ErrorCode.ERR_UNKNOWN_ERROR);
+ this.error.append("\n" + errorCodes.size() + ". " + getLastErrorCode().toString() + ", " + msg);
}
@Override
- public void flushConsole() {
- ConnectContext ctx = processor != null ? processor.getConnectContext() : ConnectContext.get();
- boolean needSend = false;
- if (error.length() > 0) {
- ctx.getState().setError("hplsql exec error, " + error.toString());
- needSend = true;
- } else if (msg.length() > 0) {
- ctx.getState().setOk(0, 0, msg.toString());
- needSend = true;
- }
- if (needSend) {
- finalizeCommand();
- reset();
- }
+ public void printError(ErrorCode errorCode, String msg) {
+ errorCodes.add(errorCode != null ? errorCode : ErrorCode.ERR_UNKNOWN_ERROR);
+ this.error.append("\n" + errorCodes.size() + ". " + getLastErrorCode().toString() + ", " + msg);
}
private void finalizeCommand() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
index 7e9ec551861..61c93adbf96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java
@@ -153,7 +153,7 @@ public class MysqlConnectProcessor extends ConnectProcessor {
}
} catch (Throwable e) {
// Catch all throwable.
- // If reach here, maybe palo bug.
+ // If reach here, maybe doris bug.
LOG.warn("Process one query failed because unknown reason: ", e);
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
e.getClass().getSimpleName() + ", msg: " + e.getMessage());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 04/04: [feat](nereids) support nullSafeEqual estimation (#31616)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 874f4c693b2b1a7baccbcaa2ab53834c30e9c73c
Author: minghong <en...@gmail.com>
AuthorDate: Fri Mar 1 19:04:06 2024 +0800
[feat](nereids) support nullSafeEqual estimation (#31616)
---
.../doris/nereids/stats/FilterEstimation.java | 15 ++++++---
.../org/apache/doris/statistics/Statistics.java | 6 +++-
.../doris/nereids/stats/FilterEstimationTest.java | 36 ++++++++++++++++++++++
3 files changed, 52 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index f7f8667e706..ad569bffd47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Like;
import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -239,7 +240,8 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
Expression left = cp.left();
Expression right = cp.right();
if (cp instanceof EqualPredicate) {
- return estimateColumnEqualToColumn(left, statsForLeft, right, statsForRight, context);
+ return estimateColumnEqualToColumn(left, statsForLeft, right, statsForRight,
+ cp instanceof NullSafeEqual, context);
}
if (cp instanceof GreaterThan || cp instanceof GreaterThanEqual) {
return estimateColumnLessThanColumn(right, statsForRight, left, statsForLeft, context);
@@ -488,7 +490,7 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
}
private Statistics estimateColumnEqualToColumn(Expression leftExpr, ColumnStatistic leftStats,
- Expression rightExpr, ColumnStatistic rightStats, EstimationContext context) {
+ Expression rightExpr, ColumnStatistic rightStats, boolean keepNull, EstimationContext context) {
StatisticRange leftRange = StatisticRange.from(leftStats, leftExpr.getDataType());
StatisticRange rightRange = StatisticRange.from(rightStats, rightExpr.getDataType());
StatisticRange leftIntersectRight = leftRange.intersect(rightRange);
@@ -497,11 +499,16 @@ public class FilterEstimation extends ExpressionVisitor<Statistics, EstimationCo
intersectBuilder.setNdv(intersect.getDistinctValues());
intersectBuilder.setMinValue(intersect.getLow());
intersectBuilder.setMaxValue(intersect.getHigh());
- intersectBuilder.setNumNulls(0);
+ double numNull = 0;
+ if (keepNull) {
+ numNull = Math.min(leftStats.numNulls, rightStats.numNulls);
+ }
+ intersectBuilder.setNumNulls(numNull);
double sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv));
- Statistics updatedStatistics = context.statistics.withSel(sel);
+ Statistics updatedStatistics = context.statistics.withSel(sel, numNull);
updatedStatistics.addColumnStats(leftExpr, intersectBuilder.build());
updatedStatistics.addColumnStats(rightExpr, intersectBuilder.build());
+
context.addKeyIfSlot(leftExpr);
context.addKeyIfSlot(rightExpr);
return updatedStatistics;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
index 1eb94dcb777..b9fa7b5541f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java
@@ -95,11 +95,15 @@ public class Statistics {
}
public Statistics withSel(double sel) {
+ return withSel(sel, 0);
+ }
+
+ public Statistics withSel(double sel, double numNull) {
sel = StatsMathUtil.minNonNaN(sel, 1);
if (Double.isNaN(rowCount)) {
return this;
}
- double newCount = rowCount * sel;
+ double newCount = rowCount * sel + numNull;
return new Statistics(newCount, widthInJoinCluster, new HashMap<>(expressionToColumnStats));
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
index 0206a093257..7ef1d8736a4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
@@ -40,6 +41,7 @@ import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Statistics;
+import org.apache.doris.statistics.StatisticsBuilder;
import com.google.common.collect.Lists;
import org.apache.commons.math3.util.Precision;
@@ -1103,4 +1105,38 @@ class FilterEstimationTest {
Assertions.assertEquals(result.getRowCount(), 10.0, 0.01);
}
+ @Test
+ public void testNullSafeEqual() {
+ ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder()
+ .setNdv(2)
+ .setAvgSizeByte(4)
+ .setNumNulls(8)
+ .setMaxValue(2)
+ .setMinValue(1)
+ .setCount(10);
+ ColumnStatistic aStats = columnStatisticBuilder.build();
+ SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+
+ columnStatisticBuilder.setNdv(2)
+ .setAvgSizeByte(4)
+ .setNumNulls(7)
+ .setMaxValue(2)
+ .setMinValue(1)
+ .setCount(10);
+ ColumnStatistic bStats = columnStatisticBuilder.build();
+ SlotReference b = new SlotReference("b", IntegerType.INSTANCE);
+
+ StatisticsBuilder statsBuilder = new StatisticsBuilder();
+ statsBuilder.setRowCount(100);
+ statsBuilder.putColumnStatistics(a, aStats);
+ statsBuilder.putColumnStatistics(b, bStats);
+
+ NullSafeEqual nse = new NullSafeEqual(a, b);
+ FilterEstimation estimator = new FilterEstimation();
+ Statistics resultNse = estimator.estimate(nse, statsBuilder.build());
+
+ EqualTo eq = new EqualTo(a, b);
+ Statistics resultEq = estimator.estimate(eq, statsBuilder.build());
+ Assertions.assertEquals(7, resultNse.getRowCount() - resultEq.getRowCount());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/04: [opt](nereids) Composite predicate supports range predicate when rewritting by materialzied view (#31538)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b26dcf26774b0f7d55aada4a1871b4b8ec3c9e05
Author: seawinde <14...@users.noreply.github.com>
AuthorDate: Fri Mar 1 18:03:10 2024 +0800
[opt](nereids) Composite predicate supports range predicate when rewritting by materialzied view (#31538)
It supports predicate composite as following:
materialized view define
> select l_shipdate, o_orderdate, l_partkey, l_suppkey
> from lineitem_1
> left join orders_1
> on lineitem_1.l_orderkey = orders_1.o_orderkey
> where l_shipdate > '2023-10-19'
the query as following can be rewritten by the materialized view above
> select l_shipdate, o_orderdate, l_partkey, l_suppkey
> from lineitem_1
> left join orders_1
> on lineitem_1.l_orderkey = orders_1.o_orderkey
> where l_shipdate > '2023-10-25'
---
.../mv/AbstractMaterializedViewRule.java | 3 +-
.../exploration/mv/MaterializedViewUtils.java | 3 +
.../nereids/rules/exploration/mv/Predicates.java | 44 ++++---
.../rules/exploration/mv/PredicatesSplitter.java | 12 ++
.../rules/expression/rules/SimplifyRange.java | 9 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 4 +
.../rules/expression/SimplifyRangeTest.java | 134 +++++++++++++++++++++
.../filter_equal_or_notequal.groovy | 104 ++++++++++------
8 files changed, 255 insertions(+), 58 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 4840416667c..a11b9101045 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -473,7 +473,8 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac
queryStructInfo,
viewStructInfo,
viewToQuerySlotMapping,
- comparisonResult);
+ comparisonResult,
+ cascadesContext);
// residual compensate
final Set<Expression> residualCompensatePredicates = Predicates.compensateResidualPredicate(
queryStructInfo,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index baf72cc278e..ce12e059b77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -75,6 +75,9 @@ public class MaterializedViewUtils {
break;
}
}
+ if (columnExpr == null) {
+ return Optional.empty();
+ }
if (!(columnExpr instanceof SlotReference)) {
return Optional.empty();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
index 472e49d3b43..139230be5d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
@@ -17,8 +17,12 @@
package org.apache.doris.nereids.rules.exploration.mv;
+import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping;
import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.rules.expression.ExpressionNormalization;
+import org.apache.doris.nereids.rules.expression.ExpressionOptimization;
+import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -143,8 +147,8 @@ public class Predicates {
public static Set<Expression> compensateRangePredicate(StructInfo queryStructInfo,
StructInfo viewStructInfo,
SlotMapping viewToQuerySlotMapping,
- ComparisonResult comparisonResult) {
- // TODO Range predicates compensate, simplify implementation currently.
+ ComparisonResult comparisonResult,
+ CascadesContext cascadesContext) {
SplitPredicate querySplitPredicate = queryStructInfo.getSplitPredicate();
SplitPredicate viewSplitPredicate = viewStructInfo.getSplitPredicate();
@@ -153,20 +157,32 @@ public class Predicates {
Expression viewRangePredicateQueryBased =
ExpressionUtils.replace(viewRangePredicate, viewToQuerySlotMapping.toSlotReferenceMap());
- Set<Expression> queryRangeSet =
- Sets.newHashSet(ExpressionUtils.extractConjunction(queryRangePredicate));
- Set<Expression> viewRangeQueryBasedSet =
- Sets.newHashSet(ExpressionUtils.extractConjunction(viewRangePredicateQueryBased));
- // remove unnecessary literal BooleanLiteral.TRUE
- queryRangeSet.remove(BooleanLiteral.TRUE);
- viewRangeQueryBasedSet.remove(BooleanLiteral.TRUE);
- // query residual predicate can not contain all view residual predicate when view have residual predicate,
- // bail out
- if (!queryRangeSet.containsAll(viewRangeQueryBasedSet)) {
+ Set<Expression> queryRangeSet = ExpressionUtils.extractConjunctionToSet(queryRangePredicate);
+ Set<Expression> viewRangeQueryBasedSet = ExpressionUtils.extractConjunctionToSet(viewRangePredicateQueryBased);
+ Set<Expression> differentExpressions = new HashSet<>();
+ Sets.difference(queryRangeSet, viewRangeQueryBasedSet).copyInto(differentExpressions);
+ Sets.difference(viewRangeQueryBasedSet, queryRangeSet).copyInto(differentExpressions);
+ // the range predicate in query and view is same, don't need to compensate
+ if (differentExpressions.isEmpty()) {
+ return differentExpressions;
+ }
+ // try to normalize the different expressions
+ Set<Expression> normalizedExpressions =
+ normalizeExpression(ExpressionUtils.and(differentExpressions), cascadesContext);
+ if (!queryRangeSet.containsAll(normalizedExpressions)) {
+ // normalized expressions is not in query, can not compensate
return null;
}
- queryRangeSet.removeAll(viewRangeQueryBasedSet);
- return queryRangeSet;
+ return normalizedExpressions;
+ }
+
+ private static Set<Expression> normalizeExpression(Expression expression, CascadesContext cascadesContext) {
+ ExpressionNormalization expressionNormalization = new ExpressionNormalization();
+ ExpressionOptimization expressionOptimization = new ExpressionOptimization();
+ ExpressionRewriteContext context = new ExpressionRewriteContext(cascadesContext);
+ expression = expressionNormalization.rewrite(expression, context);
+ expression = expressionOptimization.rewrite(expression, context);
+ return ExpressionUtils.extractConjunctionToSet(expression);
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
index da8f582f64b..f1c0ae8f96a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PredicatesSplitter.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.EqualPredicate;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.InPredicate;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
@@ -81,6 +82,17 @@ public class PredicatesSplitter {
return null;
}
+ @Override
+ public Void visitInPredicate(InPredicate inPredicate, Void context) {
+ if (containOnlyColumnRef(inPredicate.getCompareExpr(), true)
+ && (ExpressionUtils.isAllLiteral(inPredicate.getOptions()))) {
+ rangePredicates.add(inPredicate);
+ } else {
+ residualPredicates.add(inPredicate);
+ }
+ return null;
+ }
+
@Override
public Void visit(Expression expr, Void context) {
residualPredicates.add(expr);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
index c0cb834b2f4..13666405bb9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java
@@ -98,8 +98,8 @@ public class SimplifyRange extends AbstractExpressionRewriteRule {
private ValueDesc buildRange(ComparisonPredicate predicate) {
Expression rewrite = ExpressionRuleExecutor.normalize(predicate);
Expression right = rewrite.child(1);
- // only handle `NumericType`
- if (right.isLiteral() && right.getDataType().isNumericType()) {
+ // only handle `NumericType` and `DateLikeType`
+ if (right.isLiteral() && (right.getDataType().isNumericType() || right.getDataType().isDateLikeType())) {
return ValueDesc.range((ComparisonPredicate) rewrite);
}
return new UnknownValue(predicate);
@@ -132,9 +132,10 @@ public class SimplifyRange extends AbstractExpressionRewriteRule {
@Override
public ValueDesc visitInPredicate(InPredicate inPredicate, Void context) {
- // only handle `NumericType`
+ // only handle `NumericType` and `DateLikeType`
if (ExpressionUtils.isAllLiteral(inPredicate.getOptions())
- && ExpressionUtils.matchNumericType(inPredicate.getOptions())) {
+ && (ExpressionUtils.matchNumericType(inPredicate.getOptions())
+ || ExpressionUtils.matchDateLikeType(inPredicate.getOptions()))) {
return ValueDesc.discrete(inPredicate);
}
return new UnknownValue(inPredicate);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index a111edf7003..00a6ff2ec04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -467,6 +467,10 @@ public class ExpressionUtils {
return children.stream().allMatch(c -> c.getDataType().isNumericType());
}
+ public static boolean matchDateLikeType(List<Expression> children) {
+ return children.stream().allMatch(c -> c.getDataType().isDateLikeType());
+ }
+
public static boolean hasNullLiteral(List<Expression> children) {
return children.stream().anyMatch(c -> c instanceof NullLiteral);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
index 1b56e0d06fe..16476e37146 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java
@@ -158,6 +158,140 @@ public class SimplifyRangeTest {
}
+ @Test
+ public void testSimplifyDate() {
+ executor = new ExpressionRuleExecutor(ImmutableList.of(SimplifyRange.INSTANCE));
+ // assertRewrite("TA", "TA");
+ assertRewrite(
+ "(TA >= date '2024-01-01' and TA <= date '2024-01-03') or (TA > date '2024-01-05' and TA < date '2024-01-07')",
+ "(TA >= date '2024-01-01' and TA <= date '2024-01-03') or (TA > date '2024-01-05' and TA < date '2024-01-07')");
+ assertRewrite(
+ "(TA > date '2024-01-03' and TA < date '2024-01-01') or (TA > date '2024-01-07'and TA < date '2024-01-05')",
+ "FALSE");
+ assertRewrite("TA > date '2024-01-03' and TA < date '2024-01-01'", "FALSE");
+ assertRewrite("TA >= date '2024-01-01' and TA < date '2024-01-01'",
+ "TA >= date '2024-01-01' and TA < date '2024-01-01'");
+ assertRewrite("TA = date '2024-01-01' and TA > date '2024-01-10'", "FALSE");
+ assertRewrite("TA > date '2024-01-05' or TA < date '2024-01-01'",
+ "TA > date '2024-01-05' or TA < date '2024-01-01'");
+ assertRewrite("TA > date '2024-01-05' or TA > date '2024-01-01' or TA > date '2024-01-10'",
+ "TA > date '2024-01-01'");
+ assertRewrite("TA > date '2024-01-05' or TA > date '2024-01-01' or TA < date '2024-01-10'", "TA IS NOT NULL");
+ assertRewriteNotNull("TA > date '2024-01-05' or TA > date '2024-01-01' or TA < date '2024-01-10'", "TRUE");
+ assertRewrite("TA > date '2024-01-05' and TA > date '2024-01-01' and TA > date '2024-01-10'",
+ "TA > date '2024-01-10'");
+ assertRewrite("TA > date '2024-01-05' and TA > date '2024-01-01' and TA < date '2024-01-10'",
+ "TA > date '2024-01-05' and TA < date '2024-01-10'");
+ assertRewrite("TA > date '2024-01-05' or TA < date '2024-01-05'",
+ "TA > date '2024-01-05' or TA < date '2024-01-05'");
+ assertRewrite("TA > date '2024-01-01' or TA < date '2024-01-10'", "TA IS NOT NULL");
+ assertRewriteNotNull("TA > date '2024-01-01' or TA < date '2024-01-10'", "TRUE");
+ assertRewrite("TA > date '2024-01-05' and TA < date '2024-01-10'",
+ "TA > date '2024-01-05' and TA < date '2024-01-10'");
+ assertRewrite("TA > date '2024-01-05' and TA > date '2024-01-10'", "TA > date '2024-01-10'");
+ assertRewrite("(TA > date '2024-01-01' and TA > date '2024-01-10') or TA > date '2024-01-20'",
+ "TA > date '2024-01-10'");
+ assertRewrite("(TA > date '2024-01-01' or TA > date '2024-01-10') and TA > date '2024-01-20'",
+ "TA > date '2024-01-20'");
+ assertRewrite("TA > date '2024-01-05' or TA > date '2024-01-05'", "TA > date '2024-01-05'");
+ assertRewrite(
+ "(TA > date '2024-01-10' or TA > date '2024-01-20') and (TB > date '2024-01-10' and TB < date '2024-01-20')",
+ "TA > date '2024-01-10' and (TB > date '2024-01-10' and TB < date '2024-01-20') ");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') and TA > date '2024-01-10'",
+ "FALSE");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') and TA >= date '2024-01-01'",
+ "TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03')");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') and TA > date '2024-01-01'",
+ "((TA = date '2024-01-02') OR (TA = date '2024-01-03'))");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or TA >= date '2024-01-01'",
+ "TA >= date '2024-01-01'");
+ assertRewrite("TA in (date '2024-01-01')", "TA in (date '2024-01-01')");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') and TA < date '2024-01-10'",
+ "TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03')");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') and TA < date '2024-01-01'",
+ "FALSE");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or TA < date '2024-01-01'",
+ "TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03') or TA < date '2024-01-01'");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02') or TA in (date '2024-01-02', date '2024-01-03')",
+ "TA in (date '2024-01-01',date '2024-01-02',date '2024-01-03')");
+ assertRewrite("TA in (date '2024-01-01',date '2024-01-02') and TA in (date '2024-01-03', date '2024-01-04')",
+ "FALSE");
+ assertRewrite("TA = date '2024-01-03' and TA = date '2024-01-01'", "FALSE");
+ assertRewrite("TA in (date '2024-01-01') and TA in (date '2024-01-03')", "FALSE");
+ assertRewrite("TA in (date '2024-01-03') and TA in (date '2024-01-03')", "TA = date '2024-01-03'");
+ assertRewrite("(TA > date '2024-01-03' and TA < date '2024-01-01') and TB < date '2024-01-05'", "FALSE");
+ assertRewrite("(TA > date '2024-01-03' and TA < date '2024-01-01') or TB < date '2024-01-05'",
+ "TB < date '2024-01-05'");
+ }
+
+ @Test
+ public void testSimplifyDateTime() {
+ executor = new ExpressionRuleExecutor(ImmutableList.of(SimplifyRange.INSTANCE));
+ // assertRewrite("TA", "TA");
+ assertRewrite(
+ "(TA >= timestamp '2024-01-01 00:00:00' and TA <= timestamp '2024-01-03 00:00:00') or (TA > timestamp '2024-01-05 00:00:00' and TA < timestamp '2024-01-07 00:00:00')",
+ "(TA >= timestamp '2024-01-01 00:00:00' and TA <= timestamp '2024-01-03 00:00:00') or (TA > timestamp '2024-01-05 00:00:00' and TA < timestamp '2024-01-07 00:00:00')");
+ assertRewrite(
+ "(TA > timestamp '2024-01-03 00:00:10' and TA < timestamp '2024-01-01 00:00:10') or (TA > timestamp '2024-01-07 00:00:10'and TA < timestamp '2024-01-05 00:00:10')",
+ "FALSE");
+ assertRewrite("TA > timestamp '2024-01-03 00:00:10' and TA < timestamp '2024-01-01 01:00:00'", "FALSE");
+ assertRewrite("TA >= timestamp '2024-01-01 00:00:10' and TA < timestamp '2024-01-01 00:00:10'",
+ "TA >= timestamp '2024-01-01 00:00:10' and TA < timestamp '2024-01-01 00:00:10'");
+ assertRewrite("TA = timestamp '2024-01-01 10:00:10' and TA > timestamp '2024-01-10 00:00:10'", "FALSE");
+ assertRewrite("TA > timestamp '2024-01-05 00:00:10' or TA < timestamp '2024-01-01 00:00:10'",
+ "TA > timestamp '2024-01-05 00:00:10' or TA < timestamp '2024-01-01 00:00:10'");
+ assertRewrite("TA > timestamp '2024-01-05 00:00:10' or TA > timestamp '2024-01-01 00:00:10' or TA > timestamp '2024-01-10 00:00:10'",
+ "TA > timestamp '2024-01-01 00:00:10'");
+ assertRewrite("TA > timestamp '2024-01-05 00:00:10' or TA > timestamp '2024-01-01 00:00:10' or TA < timestamp '2024-01-10 00:00:10'", "TA IS NOT NULL");
+ assertRewriteNotNull("TA > timestamp '2024-01-05 00:00:10' or TA > timestamp '2024-01-01 00:00:10' or TA < timestamp '2024-01-10 00:00:10'", "TRUE");
+ assertRewrite("TA > timestamp '2024-01-05 00:00:10' and TA > timestamp '2024-01-01 00:00:10' and TA > timestamp '2024-01-10 00:00:15'",
+ "TA > timestamp '2024-01-10 00:00:15'");
+ assertRewrite("TA > timestamp '2024-01-05 00:00:10' and TA > timestamp '2024-01-01 00:00:10' and TA < timestamp '2024-01-10 00:00:10'",
+ "TA > timestamp '2024-01-05 00:00:10' and TA < timestamp '2024-01-10 00:00:10'");
+ assertRewrite("TA > timestamp '2024-01-05 00:00:10' or TA < timestamp '2024-01-05 00:00:10'",
+ "TA > timestamp '2024-01-05 00:00:10' or TA < timestamp '2024-01-05 00:00:10'");
+ assertRewrite("TA > timestamp '2024-01-01 00:02:10' or TA < timestamp '2024-01-10 00:02:10'", "TA IS NOT NULL");
+ assertRewriteNotNull("TA > timestamp '2024-01-01 00:00:00' or TA < timestamp '2024-01-10 00:00:00'", "TRUE");
+ assertRewrite("TA > timestamp '2024-01-05 01:00:00' and TA < timestamp '2024-01-10 01:00:00'",
+ "TA > timestamp '2024-01-05 01:00:00' and TA < timestamp '2024-01-10 01:00:00'");
+ assertRewrite("TA > timestamp '2024-01-05 01:00:00' and TA > timestamp '2024-01-10 01:00:00'", "TA > timestamp '2024-01-10 01:00:00'");
+ assertRewrite("(TA > timestamp '2024-01-01 01:00:00' and TA > timestamp '2024-01-10 01:00:00') or TA > timestamp '2024-01-20 01:00:00'",
+ "TA > timestamp '2024-01-10 01:00:00'");
+ assertRewrite("(TA > timestamp '2024-01-01 01:00:00' or TA > timestamp '2024-01-10 01:00:00') and TA > timestamp '2024-01-20 01:00:00'",
+ "TA > timestamp '2024-01-20 01:00:00'");
+ assertRewrite("TA > timestamp '2024-01-05 01:00:00' or TA > timestamp '2024-01-05 01:00:00'", "TA > timestamp '2024-01-05 01:00:00'");
+ assertRewrite(
+ "(TA > timestamp '2024-01-10 01:00:00' or TA > timestamp '2024-01-20 01:00:00') and (TB > timestamp '2024-01-10 01:00:00' and TB < timestamp '2024-01-20 01:00:00')",
+ "TA > timestamp '2024-01-10 01:00:00' and (TB > timestamp '2024-01-10 01:00:00' and TB < timestamp '2024-01-20 01:00:00') ");
+ assertRewrite("TA in (timestamp '2024-01-01 01:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 03:00:00') and TA > timestamp '2024-01-10 01:00:00'",
+ "FALSE");
+ assertRewrite("TA in (timestamp '2024-01-01 01:00:00',timestamp '2024-01-02 01:50:00',timestamp '2024-01-03 02:00:00') and TA >= timestamp '2024-01-01'",
+ "TA in (timestamp '2024-01-01 01:00:00',timestamp '2024-01-02 01:50:00',timestamp '2024-01-03 02:00:00')");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') and TA > timestamp '2024-01-01 02:10:00'",
+ "((TA = timestamp '2024-01-02 02:00:00') OR (TA = timestamp '2024-01-03 02:00:00'))");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') or TA >= timestamp '2024-01-01 01:00:00'",
+ "TA >= timestamp '2024-01-01 01:00:00'");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00')", "TA in (timestamp '2024-01-01 02:00:00')");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') and TA < timestamp '2024-01-10 02:00:00'",
+ "TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00')");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') and TA < timestamp '2024-01-01 02:00:00'",
+ "FALSE");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') and TA < timestamp '2024-01-01 02:00:01'",
+ "TA = timestamp '2024-01-01 02:00:00'");
+ assertRewrite("TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') or TA < timestamp '2024-01-01 01:00:00'",
+ "TA in (timestamp '2024-01-01 02:00:00',timestamp '2024-01-02 02:00:00',timestamp '2024-01-03 02:00:00') or TA < timestamp '2024-01-01 01:00:00'");
+ assertRewrite("TA in (timestamp '2024-01-01 00:00:00',timestamp '2024-01-02 00:00:00') or TA in (timestamp '2024-01-02 00:00:00', timestamp '2024-01-03 00:00:00')",
+ "TA in (timestamp '2024-01-01 00:00:00',timestamp '2024-01-02 00:00:00',timestamp '2024-01-03 00:00:00')");
+ assertRewrite("TA in (timestamp '2024-01-01 00:50:00',timestamp '2024-01-02 00:50:00') and TA in (timestamp '2024-01-03 00:50:00', timestamp '2024-01-04 00:50:00')",
+ "FALSE");
+ assertRewrite("TA = timestamp '2024-01-03 00:50:00' and TA = timestamp '2024-01-01 00:50:00'", "FALSE");
+ assertRewrite("TA in (timestamp '2024-01-01 00:50:00') and TA in (timestamp '2024-01-03 00:50:00')", "FALSE");
+ assertRewrite("TA in (timestamp '2024-01-03 00:50:00') and TA in (timestamp '2024-01-03 00:50:00')", "TA = timestamp '2024-01-03 00:50:00'");
+ assertRewrite("(TA > timestamp '2024-01-03 00:50:00' and TA < timestamp '2024-01-01 00:50:00') and TB < timestamp '2024-01-05 00:50:00'", "FALSE");
+ assertRewrite("(TA > timestamp '2024-01-03 00:50:00' and TA < timestamp '2024-01-01 00:50:00') or TB < timestamp '2024-01-05 00:50:00'",
+ "TB < timestamp '2024-01-05 00:50:00'");
+ }
+
private void assertRewrite(String expression, String expected) {
Map<String, Slot> mem = Maps.newHashMap();
Expression needRewriteExpression = replaceUnboundSlot(PARSER.parseExpression(expression), mem);
diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy
index 1422b5db585..1800a91b84b 100644
--- a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy
@@ -282,20 +282,19 @@ suite("filter_equal_or_notequal_case") {
notContains "${mv_name}(${mv_name})"
}
- // Todo: It is not currently supported and is expected to be
// mv is range and sql equal and filter in mv range
-// query_sql = """
-// select l_shipdate, o_orderdate, l_partkey, l_suppkey
-// from lineitem_1
-// left join orders_1
-// on lineitem_1.l_orderkey = orders_1.o_orderkey
-// where l_shipdate = '2023-10-19'
-// """
-// explain {
-// sql("${query_sql}")
-// contains "${mv_name}(${mv_name})"
-// }
-// compare_res(query_sql + " order by 1,2,3,4")
+ query_sql = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey
+ from lineitem_1
+ left join orders_1
+ on lineitem_1.l_orderkey = orders_1.o_orderkey
+ where l_shipdate = '2023-10-19'
+ """
+ explain {
+ sql("${query_sql}")
+ contains "${mv_name}(${mv_name})"
+ }
+ compare_res(query_sql + " order by 1,2,3,4")
// mv is range and sql is range and sql range is bigger than mv
query_sql = """
@@ -350,20 +349,19 @@ suite("filter_equal_or_notequal_case") {
notContains "${mv_name}(${mv_name})"
}
- // Todo: It is not currently supported and is expected to be
// mv is range and sql is range and sql range is bigger than mv
-// query_sql = """
-// select l_shipdate, o_orderdate, l_partkey, l_suppkey
-// from lineitem_1
-// left join orders_1
-// on lineitem_1.l_orderkey = orders_1.o_orderkey
-// where l_shipdate > '2023-10-19'
-// """
-// explain {
-// sql("${query_sql}")
-// contains "${mv_name}(${mv_name})"
-// }
-// compare_res(query_sql + " order by 1,2,3,4")
+ query_sql = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey
+ from lineitem_1
+ left join orders_1
+ on lineitem_1.l_orderkey = orders_1.o_orderkey
+ where l_shipdate > '2023-10-18'
+ """
+ explain {
+ sql("${query_sql}")
+ contains "${mv_name}(${mv_name})"
+ }
+ compare_res(query_sql + " order by 1,2,3,4")
// mv is range and sql is range and sql range is not in mv range
query_sql = """
@@ -418,20 +416,34 @@ suite("filter_equal_or_notequal_case") {
notContains "${mv_name}(${mv_name})"
}
- // Todo: It is not currently supported and is expected to be
// sql range is in mv range
-// query_sql = """
-// select l_shipdate, o_orderdate, l_partkey, l_suppkey
-// from lineitem_1
-// left join orders_1
-// on lineitem_1.l_orderkey = orders_1.o_orderkey
-// where l_shipdate in ('2023-10-18')
-// """
-// explain {
-// sql("${query_sql}")
-// contains "${mv_name}(${mv_name})"
-// }
-// compare_res(query_sql + " order by 1,2,3,4")
+ // single value
+ query_sql = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey
+ from lineitem_1
+ left join orders_1
+ on lineitem_1.l_orderkey = orders_1.o_orderkey
+ where l_shipdate in ('2023-10-18')
+ """
+ explain {
+ sql("${query_sql}")
+ contains "${mv_name}(${mv_name})"
+ }
+ compare_res(query_sql + " order by 1,2,3,4")
+
+ // multi value
+ query_sql = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey
+ from lineitem_1
+ left join orders_1
+ on lineitem_1.l_orderkey = orders_1.o_orderkey
+ where l_shipdate in ('2023-10-18', '2023-11-18')
+ """
+ explain {
+ sql("${query_sql}")
+ contains "${mv_name}(${mv_name})"
+ }
+ compare_res(query_sql + " order by 1,2,3,4")
// sql range like mv range
query_sql = """
@@ -498,6 +510,20 @@ suite("filter_equal_or_notequal_case") {
}
compare_res(query_sql + " order by 1,2,3,4")
+
+ query_sql = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey
+ from lineitem_1
+ left join orders_1
+ on lineitem_1.l_orderkey = orders_1.o_orderkey
+ where l_shipdate > '2023-10-17' and l_shipdate < '2023-10-19'
+ """
+ explain {
+ sql("${query_sql}")
+ contains "${mv_name}(${mv_name})"
+ }
+ compare_res(query_sql + " order by 1,2,3,4")
+
//
mtmv_sql = """
select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org