You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/09 07:54:33 UTC
[doris] branch master updated: [feature](Nereids) support select except syntax (#14851)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 93b281b44b [feature](Nereids) support select except syntax (#14851)
93b281b44b is described below
commit 93b281b44ba73101599c6885ad0c57afefdde8f8
Author: mch_ucchi <41...@users.noreply.github.com>
AuthorDate: Fri Dec 9 15:54:26 2022 +0800
[feature](Nereids) support select except syntax (#14851)
Support syntax: select * except(v1, v2) from t;
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 7 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 32 ++++--
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../nereids/rules/analysis/BindSlotReference.java | 7 +-
.../trees/plans/logical/LogicalProject.java | 27 ++++--
.../trees/expressions/SelectExceptTest.java | 108 +++++++++++++++++++++
.../nereids/trees/plans/PlanToStringTest.java | 2 +-
.../data/nereids_syntax_p0/select_except.out | 6 ++
.../suites/nereids_syntax_p0/select_except.groovy | 24 +++++
9 files changed, 196 insertions(+), 19 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index c734ed3dd8..1bc6f051d8 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -106,7 +106,12 @@ columnAliases
;
selectClause
- : SELECT selectHint? namedExpressionSeq
+ : SELECT selectHint? selectColumnClause
+ ;
+
+selectColumnClause
+ : namedExpressionSeq
+ | ASTERISK EXCEPT LEFT_PAREN namedExpressionSeq RIGHT_PAREN
;
whereClause
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 77c6bf4e11..4a6b38239d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -65,6 +65,7 @@ import org.apache.doris.nereids.DorisParser.QueryOrganizationContext;
import org.apache.doris.nereids.DorisParser.RegularQuerySpecificationContext;
import org.apache.doris.nereids.DorisParser.RelationContext;
import org.apache.doris.nereids.DorisParser.SelectClauseContext;
+import org.apache.doris.nereids.DorisParser.SelectColumnClauseContext;
import org.apache.doris.nereids.DorisParser.SelectHintContext;
import org.apache.doris.nereids.DorisParser.SingleStatementContext;
import org.apache.doris.nereids.DorisParser.SortClauseContext;
@@ -299,7 +300,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
SelectClauseContext selectCtx = ctx.selectClause();
LogicalPlan selectPlan;
if (ctx.fromClause() == null) {
- selectPlan = withOneRowRelation(selectCtx);
+ SelectColumnClauseContext columnCtx = selectCtx.selectColumnClause();
+ if (columnCtx.EXCEPT() != null) {
+ throw new ParseException("select-except cannot be used in one row relation", selectCtx);
+ }
+ selectPlan = withOneRowRelation(columnCtx);
} else {
LogicalPlan relation = visitFromClause(ctx.fromClause());
selectPlan = withSelectQuerySpecification(
@@ -952,8 +957,9 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
});
}
- private UnboundOneRowRelation withOneRowRelation(SelectClauseContext selectCtx) {
+ private UnboundOneRowRelation withOneRowRelation(SelectColumnClauseContext selectCtx) {
return ParserUtils.withOrigin(selectCtx, () -> {
+ // fromClause does not exists.
List<NamedExpression> projects = getNamedExpressions(selectCtx.namedExpressionSeq());
return new UnboundOneRowRelation(projects);
});
@@ -979,10 +985,11 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
// from -> where -> group by -> having -> select
LogicalPlan filter = withFilter(inputRelation, whereClause);
- LogicalPlan aggregate = withAggregate(filter, selectClause, aggClause);
+ SelectColumnClauseContext columnCtx = selectClause.selectColumnClause();
+ LogicalPlan aggregate = withAggregate(filter, columnCtx, aggClause);
// TODO: replace and process having at this position
LogicalPlan having = withHaving(aggregate, havingClause);
- return withProjection(having, selectClause, aggClause);
+ return withProjection(having, selectClause.selectColumnClause(), aggClause);
});
}
@@ -1066,15 +1073,24 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new LogicalSelectHint<>(hints, logicalPlan);
}
- private LogicalPlan withProjection(LogicalPlan input, SelectClauseContext selectCtx,
+ private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx,
Optional<AggClauseContext> aggCtx) {
return ParserUtils.withOrigin(selectCtx, () -> {
// TODO: skip if havingClause exists
if (aggCtx.isPresent()) {
return input;
} else {
- List<NamedExpression> projects = getNamedExpressions(selectCtx.namedExpressionSeq());
- return new LogicalProject<>(projects, input);
+ if (selectCtx.EXCEPT() != null) {
+ List<NamedExpression> expressions = getNamedExpressions(selectCtx.namedExpressionSeq());
+ if (!expressions.stream().allMatch(UnboundSlot.class::isInstance)) {
+ throw new ParseException("only column name is supported in except clause", selectCtx);
+ }
+ return new LogicalProject<>(ImmutableList.of(new UnboundStar(Collections.emptyList())),
+ expressions, input);
+ } else {
+ List<NamedExpression> projects = getNamedExpressions(selectCtx.namedExpressionSeq());
+ return new LogicalProject<>(projects, Collections.emptyList(), input);
+ }
}
});
}
@@ -1085,7 +1101,7 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
);
}
- private LogicalPlan withAggregate(LogicalPlan input, SelectClauseContext selectCtx,
+ private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext selectCtx,
Optional<AggClauseContext> aggCtx) {
return input.optionalMap(aggCtx, () -> {
GroupingElementContext groupingElementContext = aggCtx.get().groupingElement();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 4c4d5f43b1..e3e12f11f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -71,6 +71,8 @@ public enum RuleType {
CHECK_ROW_POLICY(RuleTypeClass.REWRITE),
+ ELIMINATE_EXCEPT(RuleTypeClass.REWRITE),
+
// check analysis rule
CHECK_ANALYSIS(RuleTypeClass.CHECK),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
index df6f9a2232..12743deb37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
@@ -102,8 +102,11 @@ public class BindSlotReference implements AnalysisRuleFactory {
LogicalProject<GroupPlan> project = ctx.root;
List<NamedExpression> boundSlots =
bind(project.getProjects(), project.children(), project, ctx.cascadesContext);
- List<NamedExpression> newOutput = adjustNullableForProjects(project, boundSlots);
- return new LogicalProject<>(flatBoundStar(newOutput), project.child());
+ List<NamedExpression> exceptSlots = bind(project.getExcepts(), project.children(), project,
+ ctx.cascadesContext);
+ List<NamedExpression> newOutput = flatBoundStar(adjustNullableForProjects(project, boundSlots));
+ newOutput.removeAll(exceptSlots);
+ return new LogicalProject<>(newOutput, project.child());
})
),
RuleType.BINDING_FILTER_SLOT.build(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
index f49a27da2c..7280312f39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java
@@ -31,6 +31,7 @@ import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -41,9 +42,14 @@ import java.util.Optional;
public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYPE> implements Project {
private final ImmutableList<NamedExpression> projects;
+ private final ImmutableList<NamedExpression> excepts;
+
+ public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts, CHILD_TYPE child) {
+ this(projects, excepts, Optional.empty(), Optional.empty(), child);
+ }
public LogicalProject(List<NamedExpression> projects, CHILD_TYPE child) {
- this(projects, Optional.empty(), Optional.empty(), child);
+ this(projects, Collections.emptyList(), child);
}
/**
@@ -51,10 +57,12 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
*
* @param projects project list
*/
- public LogicalProject(List<NamedExpression> projects, Optional<GroupExpression> groupExpression,
- Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
+ public LogicalProject(List<NamedExpression> projects, List<NamedExpression> excepts,
+ Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
+ CHILD_TYPE child) {
super(PlanType.LOGICAL_PROJECT, groupExpression, logicalProperties, child);
this.projects = ImmutableList.copyOf(Objects.requireNonNull(projects, "projects can not be null"));
+ this.excepts = ImmutableList.copyOf(excepts);
}
/**
@@ -67,6 +75,10 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
return projects;
}
+ public List<NamedExpression> getExcepts() {
+ return excepts;
+ }
+
@Override
public List<Slot> computeOutput() {
return projects.stream()
@@ -77,7 +89,8 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
@Override
public String toString() {
return Utils.toSqlString("LogicalProject",
- "projects", projects
+ "projects", projects,
+ "excepts", excepts
);
}
@@ -111,16 +124,16 @@ public class LogicalProject<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_
@Override
public LogicalUnary<Plan> withChildren(List<Plan> children) {
Preconditions.checkArgument(children.size() == 1);
- return new LogicalProject<>(projects, children.get(0));
+ return new LogicalProject<>(projects, excepts, children.get(0));
}
@Override
public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
- return new LogicalProject<>(projects, groupExpression, Optional.of(getLogicalProperties()), child());
+ return new LogicalProject<>(projects, excepts, groupExpression, Optional.of(getLogicalProperties()), child());
}
@Override
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
- return new LogicalProject<>(projects, Optional.empty(), logicalProperties, child());
+ return new LogicalProject<>(projects, excepts, Optional.empty(), logicalProperties, child());
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java
new file mode 100644
index 0000000000..23606f95f8
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/SelectExceptTest.java
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.analyzer.UnboundSlot;
+import org.apache.doris.nereids.analyzer.UnboundStar;
+import org.apache.doris.nereids.exceptions.ParseException;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanConstructor;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class SelectExceptTest implements PatternMatchSupported {
+ @Test
+ public void testExcept() {
+ LogicalOlapScan olapScan = PlanConstructor.newLogicalOlapScan(0, "t1", 1);
+ LogicalProject<LogicalOlapScan> project = new LogicalProject<>(
+ ImmutableList.of(new UnboundStar(ImmutableList.of("db", "t1"))),
+ ImmutableList.of(new UnboundSlot("db", "t1", "id")),
+ olapScan);
+ PlanChecker.from(MemoTestUtils.createConnectContext())
+ .analyze(project)
+ .matches(
+ logicalProject(
+ logicalOlapScan()
+ ).when(proj -> proj.getExcepts().isEmpty() && proj.getProjects().size() == 1)
+ );
+ }
+
+ @Test
+ public void testParse() {
+ String sql1 = "select * except(v1, v2) from t1";
+ PlanChecker.from(MemoTestUtils.createConnectContext())
+ .checkParse(sql1, (checker) -> checker.matches(
+ logicalProject(
+ logicalCheckPolicy(
+ unboundRelation()
+ )
+ ).when(project -> project.getExcepts().size() == 2
+ && project.getProjects().get(0) instanceof UnboundStar)
+ ));
+
+ String sql2 = "select k1, k2, v1, v2 except(v1, v2) from t1";
+ Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext())
+ .checkParse(sql2, (checker) -> checker.matches(
+ logicalProject(
+ logicalCheckPolicy(
+ unboundRelation()
+ )
+ ).when(project -> project.getExcepts().size() == 2
+ && project.getProjects().get(0) instanceof UnboundStar)
+ )));
+
+ String sql3 = "select * except(v1, v2)";
+ Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext())
+ .checkParse(sql3, (checker) -> checker.matches(
+ logicalProject(
+ logicalCheckPolicy(
+ unboundRelation()
+ )
+ ).when(project -> project.getExcepts().size() == 2
+ && project.getProjects().get(0) instanceof UnboundStar)
+ )));
+
+ String sql4 = "select * except() from t1";
+ Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext())
+ .checkParse(sql4, (checker) -> checker.matches(
+ logicalProject(
+ logicalCheckPolicy(
+ unboundRelation()
+ )
+ ).when(project -> project.getExcepts().size() == 2
+ && project.getProjects().get(0) instanceof UnboundStar)
+ )));
+
+ String sql5 = "select * except(v1 + v2, v3 as k3) from t1";
+ Assertions.assertThrows(ParseException.class, () -> PlanChecker.from(MemoTestUtils.createConnectContext())
+ .checkParse(sql5, (checker) -> checker.matches(
+ logicalProject(
+ logicalCheckPolicy(
+ unboundRelation()
+ )
+ ).when(project -> project.getExcepts().size() == 2
+ && project.getProjects().get(0) instanceof UnboundStar)
+ )));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
index 1dc0a8047a..97710b3713 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java
@@ -89,7 +89,7 @@ public class PlanToStringTest {
LogicalProject<Plan> plan = new LogicalProject<>(ImmutableList.of(
new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList())), child);
- Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+] \\)"));
+ Assertions.assertTrue(plan.toString().matches("LogicalProject \\( projects=\\[a#\\d+], excepts=\\[] \\)"));
}
@Test
diff --git a/regression-test/data/nereids_syntax_p0/select_except.out b/regression-test/data/nereids_syntax_p0/select_except.out
new file mode 100644
index 0000000000..3314edd5ec
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/select_except.out
@@ -0,0 +1,6 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+9 ,gJ6K2MKveYxQT IRAN 6 IRAN MIDDLE EAST 20-338-906-3675
+15 DF35PepL5saAK INDIA 0 INDIA ASIA 18-687-542-7601
+29 VVSymB3fbwaN ARGENTINA4 ARGENTINA AMERICA 11-773-203-7342
+
diff --git a/regression-test/suites/nereids_syntax_p0/select_except.groovy b/regression-test/suites/nereids_syntax_p0/select_except.groovy
new file mode 100644
index 0000000000..88ec826482
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/select_except.groovy
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("select_except") {
+ sql "SET enable_vectorized_engine=true"
+ sql "SET enable_nereids_planner=true"
+ sql "SET enable_fallback_to_original_planner=false"
+
+ qt_select "select * except (s_name) from supplier order by s_suppkey"
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org