You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/22 07:45:20 UTC
[doris] branch branch-1.2-lts updated (f4409d3c87 -> 9c0239ef40)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from f4409d3c87 [bugfix](thirdparty) patch simdjson to avoid conflict with odbc macro BOOL (#15223)
new e7589b8367 [feature](planner) compact multi-euqals to in-predicate #14876
new dcfcedaefa [fix](join)the policy to choose colocate join is not correct (#15140)
new 1bf34ee4ce [chore](routine load) remove deprecated property of librdkafka reconnect.backoff.jitter.ms #15172
new 9c0239ef40 [fix](jdbc) fix create table like table of jdbc error (#15179)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/runtime/routine_load/data_consumer.cpp | 3 +-
.../java/org/apache/doris/analysis/Analyzer.java | 2 +
.../main/java/org/apache/doris/catalog/Env.java | 3 +-
.../apache/doris/planner/DistributedPlanner.java | 2 +-
.../org/apache/doris/planner/HashJoinNode.java | 13 ++
.../java/org/apache/doris/planner/PlanNode.java | 19 ++-
.../rewrite/CompactEqualsToInPredicateRule.java | 160 +++++++++++++++++++++
.../doris/analysis/PartitionPruneTestBase.java | 2 -
.../CompactEqualsToInPredicateRuleTest.java | 116 +++++++++++++++
.../data/performance_p0/redundant_conjuncts.out | 2 +-
.../correctness_p0/test_colocate_join.groovy | 84 +++++++++++
11 files changed, 398 insertions(+), 8 deletions(-)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
create mode 100644 fe/fe-core/src/test/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRuleTest.java
create mode 100644 regression-test/suites/correctness_p0/test_colocate_join.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/04: [feature](planner) compact multi-euqals to in-predicate #14876
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e7589b83675f37092684af8b7b793e521d872477
Author: minghong <mi...@163.com>
AuthorDate: Tue Dec 20 09:43:34 2022 +0800
[feature](planner) compact multi-euqals to in-predicate #14876
---
.../java/org/apache/doris/analysis/Analyzer.java | 2 +
.../rewrite/CompactEqualsToInPredicateRule.java | 160 +++++++++++++++++++++
.../doris/analysis/PartitionPruneTestBase.java | 2 -
.../CompactEqualsToInPredicateRuleTest.java | 116 +++++++++++++++
.../data/performance_p0/redundant_conjuncts.out | 2 +-
5 files changed, 279 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index a34146c607..85289d9044 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -46,6 +46,7 @@ import org.apache.doris.planner.PlanNode;
import org.apache.doris.planner.RuntimeFilter;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.BetweenToCompoundRule;
+import org.apache.doris.rewrite.CompactEqualsToInPredicateRule;
import org.apache.doris.rewrite.CompoundPredicateWriteRule;
import org.apache.doris.rewrite.ExprRewriteRule;
import org.apache.doris.rewrite.ExprRewriter;
@@ -410,6 +411,7 @@ public class Analyzer {
rules.add(RewriteEncryptKeyRule.INSTANCE);
rules.add(RewriteInPredicateRule.INSTANCE);
rules.add(RewriteAliasFunctionRule.INSTANCE);
+ rules.add(CompactEqualsToInPredicateRule.INSTANCE);
List<ExprRewriteRule> onceRules = Lists.newArrayList();
onceRules.add(ExtractCommonFactorsRule.INSTANCE);
onceRules.add(InferFiltersRule.INSTANCE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
new file mode 100644
index 0000000000..7375b83121
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRule.java
@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.rewrite;
+
+import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.analysis.CompoundPredicate.Operator;
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Pair;
+import org.apache.doris.rewrite.ExprRewriter.ClauseType;
+
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+/*
+a = 1 or a = 2 or a = 3 or a in (4, 5, 6) => a in (1, 2, 3, 4, 5, 6)
+ */
+public class CompactEqualsToInPredicateRule implements ExprRewriteRule {
+ public static CompactEqualsToInPredicateRule INSTANCE = new CompactEqualsToInPredicateRule();
+ private static final int COMPACT_SIZE = 2;
+
+ @Override
+ public Expr apply(Expr expr, Analyzer analyzer, ClauseType clauseType) throws AnalysisException {
+ if (expr == null) {
+ return expr;
+ }
+ if (expr instanceof CompoundPredicate) {
+ CompoundPredicate comp = (CompoundPredicate) expr;
+ if (comp.getOp() == Operator.OR) {
+ Pair<Boolean, Expr> compactResult = compactEqualsToInPredicate(expr);
+ if (compactResult.first) {
+ return compactResult.second;
+ }
+ }
+ }
+ return expr;
+ }
+
+ /*
+ expr in form of A or B or ...
+ */
+ private Pair<Boolean, Expr> compactEqualsToInPredicate(Expr expr) {
+ boolean changed = false;
+ List<Expr> disConjuncts = getDisconjuncts(expr);
+ if (disConjuncts.size() < COMPACT_SIZE) {
+ return Pair.of(false, expr);
+ }
+ Map<SlotRef, Set<Expr>> equalMap = new HashMap<>();
+ Map<SlotRef, InPredicate> inPredMap = new HashMap<>();
+ Expr result = null;
+ for (Expr disConj : disConjuncts) {
+ if (disConj instanceof BinaryPredicate
+ && ((BinaryPredicate) disConj).getOp() == BinaryPredicate.Operator.EQ) {
+ BinaryPredicate binary = (BinaryPredicate) disConj;
+ if (binary.getChild(0) instanceof SlotRef
+ && binary.getChild(1) instanceof LiteralExpr) {
+ equalMap.computeIfAbsent((SlotRef) binary.getChild(0), k -> new HashSet<>());
+ equalMap.get((SlotRef) binary.getChild(0)).add((LiteralExpr) binary.getChild(1));
+ } else if (binary.getChild(0) instanceof LiteralExpr
+ && binary.getChild(1) instanceof SlotRef) {
+ equalMap.computeIfAbsent((SlotRef) binary.getChild(1), k -> new HashSet<>());
+ equalMap.get((SlotRef) binary.getChild(1)).add((LiteralExpr) binary.getChild(0));
+ } else {
+ result = addDisconjunct(result, disConj);
+ }
+ } else if (disConj instanceof InPredicate && !((InPredicate) disConj).isNotIn()) {
+ InPredicate in = (InPredicate) disConj;
+ Expr compareExpr = in.getChild(0);
+ if (compareExpr instanceof SlotRef) {
+ SlotRef slot = (SlotRef) compareExpr;
+ InPredicate val = inPredMap.get(slot);
+ if (val == null) {
+ inPredMap.put(slot, in);
+ } else {
+ val.getChildren().addAll(in.getListChildren());
+ inPredMap.put(slot, val);
+ }
+ }
+ } else {
+ result = addDisconjunct(result, disConj);
+ }
+ }
+
+ for (Entry<SlotRef, Set<Expr>> entry : equalMap.entrySet()) {
+ SlotRef slot = entry.getKey();
+ InPredicate in = inPredMap.get(slot);
+ if (entry.getValue().size() >= COMPACT_SIZE || in != null) {
+ if (in == null) {
+ in = new InPredicate(entry.getKey(), Lists.newArrayList(entry.getValue()), false);
+ inPredMap.put(slot, in);
+ } else {
+ in.getChildren().addAll(Lists.newArrayList(entry.getValue()));
+ }
+ changed = true;
+ } else {
+ for (Expr right : entry.getValue()) {
+ result = addDisconjunct(result,
+ new BinaryPredicate(BinaryPredicate.Operator.EQ,
+ entry.getKey(),
+ right));
+ }
+ }
+ }
+ for (InPredicate in : inPredMap.values()) {
+ result = addDisconjunct(result, in);
+ }
+ return Pair.of(changed, result);
+ }
+
+ private Expr addDisconjunct(Expr result, Expr conj) {
+ if (result == null) {
+ return conj;
+ } else {
+ return new CompoundPredicate(Operator.OR, result, conj);
+ }
+ }
+
+ private List<Expr> getDisconjuncts(Expr expr) {
+ List<Expr> result = new ArrayList<>();
+ if (expr instanceof CompoundPredicate) {
+ CompoundPredicate comp = ((CompoundPredicate) expr);
+ if (comp.getOp() == Operator.OR) {
+ result.addAll(getDisconjuncts(comp.getChild(0)));
+ result.addAll(getDisconjuncts(comp.getChild(1)));
+ } else {
+ result.add(expr);
+ }
+ } else {
+ result.add(expr);
+ }
+ return result;
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
index 95c4790bea..7ff8e2fda1 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
@@ -29,8 +29,6 @@ public abstract class PartitionPruneTestBase extends TestWithFeService {
protected void doTest() throws Exception {
for (RangePartitionPruneTest.TestCase testCase : cases) {
- connectContext.getSessionVariable().partitionPruneAlgorithmVersion = 1;
- assertExplainContains(1, testCase.sql, testCase.v1Result);
connectContext.getSessionVariable().partitionPruneAlgorithmVersion = 2;
assertExplainContains(2, testCase.sql, testCase.v2Result);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRuleTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRuleTest.java
new file mode 100644
index 0000000000..baa694ea42
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/CompactEqualsToInPredicateRuleTest.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.rewrite;
+
+import org.apache.doris.analysis.BinaryPredicate;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.analysis.CompoundPredicate.Operator;
+import org.apache.doris.analysis.InPredicate;
+import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.jmockit.Deencapsulation;
+import org.apache.doris.datasource.InternalCatalog;
+
+import com.google.common.collect.Lists;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+
+public class CompactEqualsToInPredicateRuleTest {
+ private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME;
+
+ //a=1 or b=2 or a=3 or b=4
+ //=> a in (1, 2) or b in (3, 4)
+ @Test
+ public void testCompactEquals() {
+ SlotRef a = new SlotRef(new TableName(internalCtl, "db1", "tb1"), "a");
+ SlotRef b = new SlotRef(new TableName(internalCtl, "db1", "tb1"), "b");
+ IntLiteral i1 = new IntLiteral(1);
+ IntLiteral i2 = new IntLiteral(2);
+ IntLiteral i3 = new IntLiteral(3);
+ IntLiteral i4 = new IntLiteral(4);
+ BinaryPredicate aeq1 = new BinaryPredicate(BinaryPredicate.Operator.EQ, i1, a);
+ BinaryPredicate aeq3 = new BinaryPredicate(BinaryPredicate.Operator.EQ, a, i3);
+ BinaryPredicate beq2 = new BinaryPredicate(BinaryPredicate.Operator.EQ, b, i2);
+ BinaryPredicate beq4 = new BinaryPredicate(BinaryPredicate.Operator.EQ, b, i4);
+ CompoundPredicate or1 = new CompoundPredicate(Operator.OR, aeq1, beq2);
+ CompoundPredicate or2 = new CompoundPredicate(Operator.OR, or1, aeq3);
+ CompoundPredicate or3 = new CompoundPredicate(Operator.OR, or2, beq4);
+ CompactEqualsToInPredicateRule rule = new CompactEqualsToInPredicateRule();
+ Pair result = Deencapsulation.invoke(rule,
+ "compactEqualsToInPredicate", or3);
+ Assertions.assertEquals(true, result.first);
+ Assertions.assertTrue(result.second instanceof CompoundPredicate);
+ CompoundPredicate or = (CompoundPredicate) result.second;
+ Assertions.assertEquals(Operator.OR, or.getOp());
+ InPredicate in1 = (InPredicate) or.getChild(0);
+ InPredicate in2 = (InPredicate) or.getChild(1);
+ SlotRef s1 = (SlotRef) in1.getChildren().get(0);
+ InPredicate tmp;
+ if (s1.getColumnName().equals("b")) {
+ tmp = in1;
+ in1 = in2;
+ in2 = tmp;
+ }
+ Assertions.assertEquals(in1.getChild(0), a);
+ Assertions.assertEquals(in2.getChild(0), b);
+
+ HashSet<IntLiteral> seta = new HashSet<>();
+ seta.add(i1);
+ seta.add(i3);
+ HashSet<IntLiteral> setb = new HashSet<>();
+ setb.add(i2);
+ setb.add(i4);
+
+ Assertions.assertTrue(seta.contains(in1.getChild(1)));
+ Assertions.assertTrue(seta.contains(in1.getChild(2)));
+
+ Assertions.assertTrue(setb.contains(in2.getChild(1)));
+ Assertions.assertTrue(setb.contains(in2.getChild(2)));
+ }
+
+ //a=1 or a in (3, 2) => a in (1, 2, 3)
+ @Test
+ public void testCompactEqualsAndIn() {
+ SlotRef a = new SlotRef(new TableName(internalCtl, "db1", "tb1"), "a");
+ IntLiteral i1 = new IntLiteral(1);
+ IntLiteral i2 = new IntLiteral(2);
+ IntLiteral i3 = new IntLiteral(3);
+ BinaryPredicate aeq1 = new BinaryPredicate(BinaryPredicate.Operator.EQ, i1, a);
+ InPredicate ain23 = new InPredicate(a, Lists.newArrayList(i2, i3), false);
+ CompoundPredicate or1 = new CompoundPredicate(Operator.OR, aeq1, ain23);
+ CompactEqualsToInPredicateRule rule = new CompactEqualsToInPredicateRule();
+ Pair result = Deencapsulation.invoke(rule,
+ "compactEqualsToInPredicate", or1);
+ Assertions.assertEquals(true, result.first);
+ Assertions.assertTrue(result.second instanceof InPredicate);
+ InPredicate in123 = (InPredicate) result.second;
+ Assertions.assertEquals(in123.getChild(0), a);
+ HashSet<IntLiteral> seta = new HashSet<>();
+ seta.add(i1);
+ seta.add(i2);
+ seta.add(i3);
+
+ Assertions.assertTrue(seta.contains(in123.getChild(1)));
+ Assertions.assertTrue(seta.contains(in123.getChild(2)));
+ Assertions.assertTrue(seta.contains(in123.getChild(3)));
+ }
+}
diff --git a/regression-test/data/performance_p0/redundant_conjuncts.out b/regression-test/data/performance_p0/redundant_conjuncts.out
index 98178f31aa..9cba503956 100644
--- a/regression-test/data/performance_p0/redundant_conjuncts.out
+++ b/regression-test/data/performance_p0/redundant_conjuncts.out
@@ -23,7 +23,7 @@ PLAN FRAGMENT 0
0:VOlapScanNode
TABLE: default_cluster:regression_test_performance_p0.redundant_conjuncts(redundant_conjuncts), PREAGGREGATION: OFF. Reason: No AggregateInfo
- PREDICATES: (`k1` = 1 OR `k1` = 2)
+ PREDICATES: `k1` IN (2, 1)
partitions=0/1, tablets=0/0, tabletList=
cardinality=0, avgRowSize=8.0, numNodes=1
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/04: [chore](routine load) remove deprecated property of librdkafka reconnect.backoff.jitter.ms #15172
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1bf34ee4ce6949e4efe193bb8db4193aaefe0568
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Tue Dec 20 10:13:56 2022 +0800
[chore](routine load) remove deprecated property of librdkafka reconnect.backoff.jitter.ms #15172
---
be/src/runtime/routine_load/data_consumer.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/be/src/runtime/routine_load/data_consumer.cpp b/be/src/runtime/routine_load/data_consumer.cpp
index 60a0f93a44..c88a61c8ba 100644
--- a/be/src/runtime/routine_load/data_consumer.cpp
+++ b/be/src/runtime/routine_load/data_consumer.cpp
@@ -76,7 +76,8 @@ Status KafkaDataConsumer::init(StreamLoadContext* ctx) {
RETURN_IF_ERROR(set_conf("statistics.interval.ms", "0"));
RETURN_IF_ERROR(set_conf("auto.offset.reset", "error"));
RETURN_IF_ERROR(set_conf("socket.keepalive.enable", "true"));
- RETURN_IF_ERROR(set_conf("reconnect.backoff.jitter.ms", "100"));
+ RETURN_IF_ERROR(set_conf("reconnect.backoff.ms", "100"));
+ RETURN_IF_ERROR(set_conf("reconnect.backoff.max.ms", "10000"));
RETURN_IF_ERROR(set_conf("api.version.request", config::kafka_api_version_request));
RETURN_IF_ERROR(set_conf("api.version.fallback.ms", "0"));
RETURN_IF_ERROR(set_conf("broker.version.fallback", config::kafka_broker_version_fallback));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/04: [fix](join)the policy to choose colocate join is not correct (#15140)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit dcfcedaefa573b4e348f5ff7442bb21895065eba
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Tue Dec 20 09:44:47 2022 +0800
[fix](join)the policy to choose colocate join is not correct (#15140)
* [hotfix](dev-1.0.1) fix colocate join bug in vec engine after introducing output tuple (#10651)
to support vectorized outer join, we introduced a out tuple for hash join node,
but it breaks the checking for colocate join.
To solve this problem, we need map the output slot id to the children's slot id of hash join node,
and the colocate join can be checked correctly.
* fix colocate join bug
* fix non vec colocate join issue
Co-authored-by: lichi <li...@rateup.com.cn>
* add test cases
Co-authored-by: lichi <li...@rateup.com.cn>
---
.../apache/doris/planner/DistributedPlanner.java | 2 +-
.../org/apache/doris/planner/HashJoinNode.java | 13 ++++
.../java/org/apache/doris/planner/PlanNode.java | 19 ++++-
.../correctness_p0/test_colocate_join.groovy | 84 ++++++++++++++++++++++
4 files changed, 115 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
index e3bd99c064..6382787d1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
@@ -489,7 +489,7 @@ public class DistributedPlanner {
return null;
}
ScanNode scanNode = planFragment.getPlanRoot()
- .getScanNodeInOneFragmentByTupleId(slotRef.getDesc().getParent().getId());
+ .getScanNodeInOneFragmentBySlotRef(slotRef);
if (scanNode == null) {
cannotReason.add(DistributedPlanColocateRule.REDISTRIBUTED_SRC_DATA);
return null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
index 8eb8e85256..a48778bccd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java
@@ -841,4 +841,17 @@ public class HashJoinNode extends JoinNodeBase {
public void setOtherJoinConjuncts(List<Expr> otherJoinConjuncts) {
this.otherJoinConjuncts = otherJoinConjuncts;
}
+
+ SlotRef getMappedInputSlotRef(SlotRef slotRef) {
+ if (outputSmap != null) {
+ Expr mappedExpr = outputSmap.mappingForRhsExpr(slotRef);
+ if (mappedExpr != null && mappedExpr instanceof SlotRef) {
+ return (SlotRef) mappedExpr;
+ } else {
+ return null;
+ }
+ } else {
+ return slotRef;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index 4ae249f4cf..2c3baf6ec8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -28,6 +28,7 @@ import org.apache.doris.analysis.ExprId;
import org.apache.doris.analysis.ExprSubstitutionMap;
import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.SlotId;
+import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.TupleDescriptor;
import org.apache.doris.analysis.TupleId;
import org.apache.doris.catalog.Function;
@@ -897,12 +898,26 @@ public abstract class PlanNode extends TreeNode<PlanNode> implements PlanStats {
return sb.toString();
}
- public ScanNode getScanNodeInOneFragmentByTupleId(TupleId tupleId) {
+ public ScanNode getScanNodeInOneFragmentBySlotRef(SlotRef slotRef) {
+ TupleId tupleId = slotRef.getDesc().getParent().getId();
if (this instanceof ScanNode && tupleIds.contains(tupleId)) {
return (ScanNode) this;
+ } else if (this instanceof HashJoinNode) {
+ HashJoinNode hashJoinNode = (HashJoinNode) this;
+ SlotRef inputSlotRef = hashJoinNode.getMappedInputSlotRef(slotRef);
+ if (inputSlotRef != null) {
+ for (PlanNode planNode : children) {
+ ScanNode scanNode = planNode.getScanNodeInOneFragmentBySlotRef(inputSlotRef);
+ if (scanNode != null) {
+ return scanNode;
+ }
+ }
+ } else {
+ return null;
+ }
} else if (!(this instanceof ExchangeNode)) {
for (PlanNode planNode : children) {
- ScanNode scanNode = planNode.getScanNodeInOneFragmentByTupleId(tupleId);
+ ScanNode scanNode = planNode.getScanNodeInOneFragmentBySlotRef(slotRef);
if (scanNode != null) {
return scanNode;
}
diff --git a/regression-test/suites/correctness_p0/test_colocate_join.groovy b/regression-test/suites/correctness_p0/test_colocate_join.groovy
new file mode 100644
index 0000000000..6b1e81eb80
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_colocate_join.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_colocate_join") {
+ sql """ DROP TABLE IF EXISTS `test_colo1` """
+ sql """ DROP TABLE IF EXISTS `test_colo2` """
+ sql """ DROP TABLE IF EXISTS `test_colo3` """
+ sql """
+ CREATE TABLE `test_colo1` (
+ `id` varchar(64) NULL,
+ `name` varchar(64) NULL,
+ `age` int NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`,`name`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "colocate_with" = "group",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+ sql """
+ CREATE TABLE `test_colo2` (
+ `id` varchar(64) NULL,
+ `name` varchar(64) NULL,
+ `age` int NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`,`name`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "colocate_with" = "group",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """
+ CREATE TABLE `test_colo3` (
+ `id` varchar(64) NULL,
+ `name` varchar(64) NULL,
+ `age` int NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`id`,`name`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`id`,`name`) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "colocate_with" = "group",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """insert into test_colo1 values('1','a',12);"""
+ sql """insert into test_colo2 values('1','a',12);"""
+ sql """insert into test_colo3 values('1','a',12);"""
+
+ explain {
+ sql("select a.id,a.name,b.id,b.name,c.id,c.name from test_colo1 a inner join test_colo2 b on a.id = b.id and a.name = b.name inner join test_colo3 c on a.id=c.id and a.name= c.name")
+ contains "4:VHASH JOIN\n | join op: INNER JOIN(COLOCATE[])[]"
+ contains "2:VHASH JOIN\n | join op: INNER JOIN(COLOCATE[])[]"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/04: [fix](jdbc) fix create table like table of jdbc error (#15179)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9c0239ef40506e791c21f03aaf25ed48853cb9c2
Author: xueweizhang <zx...@163.com>
AuthorDate: Wed Dec 21 08:56:43 2022 +0800
[fix](jdbc) fix create table like table of jdbc error (#15179)
when create table like table of jdbc, it will get error like
'errCode = 2, detailMessage = Failed to execute CREATE TABLE LIKE baseall_mysql.
Reason: errCode = 2, detailMessage = property table_type must be set'
this pr fix it.
---
fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index af2393e9e1..da7df2caef 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3137,7 +3137,8 @@ public class Env {
addTableComment(jdbcTable, sb);
sb.append("\nPROPERTIES (\n");
sb.append("\"resource\" = \"").append(jdbcTable.getResourceName()).append("\",\n");
- sb.append("\"table\" = \"").append(jdbcTable.getJdbcTable()).append("\"");
+ sb.append("\"table\" = \"").append(jdbcTable.getJdbcTable()).append("\",\n");
+ sb.append("\"table_type\" = \"").append(jdbcTable.getJdbcTypeName()).append("\"");
sb.append("\n)");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org