You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2017/09/25 19:01:34 UTC
asterixdb git commit: [ASTERIXDB-2105][COMP] Support broadcast join
hint with multiple predicates
Repository: asterixdb
Updated Branches:
refs/heads/master 9ee6f2d7c -> 88b154dfd
[ASTERIXDB-2105][COMP] Support broadcast join hint with multiple predicates
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Broadcast hint now works if join condition has multiple predicates.
Only one predicate needs to be annotated.
Change-Id: Id1475c5e574be2632c92ae619f3a36c56c222514
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2024
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Xikui Wang <xk...@gmail.com>
Reviewed-by: Till Westmann <ti...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/88b154df
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/88b154df
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/88b154df
Branch: refs/heads/master
Commit: 88b154dfd66d1641d0fe25fe4e0ff9e54c24dcad
Parents: 9ee6f2d
Author: Dmitry Lychagin <dm...@couchbase.com>
Authored: Fri Sep 22 16:25:49 2017 -0700
Committer: Dmitry Lychagin <dm...@couchbase.com>
Committed: Mon Sep 25 12:01:14 2017 -0700
----------------------------------------------------------------------
.../asterix/test/optimizer/OptimizerTest.java | 2 +
.../queries/hints/broadcast_hint_1.sqlpp | 46 ++++++++++++++
.../queries/hints/broadcast_hint_2.sqlpp | 46 ++++++++++++++
.../queries/hints/broadcast_hint_3.sqlpp | 46 ++++++++++++++
.../results/hints/broadcast_hint_1.plan | 23 +++++++
.../results/hints/broadcast_hint_2.plan | 23 +++++++
.../results/hints/broadcast_hint_3.plan | 23 +++++++
.../algebricks/rewriter/util/JoinUtils.java | 67 ++++++++++++--------
8 files changed, 250 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index 409bbdc..d1c3ce4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -172,6 +172,8 @@ public class OptimizerTest {
LOGGER.info("RUN TEST: \"" + queryFile.getPath() + "\"");
Reader query = new BufferedReader(new InputStreamReader(new FileInputStream(queryFile), "UTF-8"));
+ LOGGER.info("ACTUAL RESULT FILE: " + actualFile.getAbsolutePath());
+
// Forces the creation of actualFile.
actualFile.getParentFile().mkdirs();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_1.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_1.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_1.sqlpp
new file mode 100644
index 0000000..02ba35d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_1.sqlpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Test broadcast join hint.
+* Hint on one conjunct
+* Expected Res : Success
+* Date : 09/22/2017
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type TT as closed {
+ c_key : integer,
+ c_1 : integer,
+ c_2 : integer,
+ c_3 : integer,
+ c_4 : integer
+};
+
+create dataset t1(TT) primary key c_key;
+
+create dataset t2(TT) primary key c_key;
+
+select t1.c_4 as c0, t2.c_4 as c1
+from t1 join t2
+on t1.c_1 = t2.c_1 and t1.c_2 /*+ bcast */ = t2.c_2 and t1.c_3 = t2.c_3
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_2.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_2.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_2.sqlpp
new file mode 100644
index 0000000..1feb21f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_2.sqlpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Test broadcast join hint
+* Hint on each conjunct
+* Expected Res : Success
+* Date : 09/22/2017
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type TT as closed {
+ c_key : integer,
+ c_1 : integer,
+ c_2 : integer,
+ c_3 : integer,
+ c_4 : integer
+};
+
+create dataset t1(TT) primary key c_key;
+
+create dataset t2(TT) primary key c_key;
+
+select t1.c_4 as c0, t2.c_4 as c1
+from t1 join t2
+on t1.c_1 /*+ bcast */ = t2.c_1 and t1.c_2 /*+ bcast */ = t2.c_2 and t1.c_3 /*+ bcast */ = t2.c_3
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_3.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_3.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_3.sqlpp
new file mode 100644
index 0000000..a3fd4f6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/hints/broadcast_hint_3.sqlpp
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+* Description : Test broadcast join hint
+* Incompatible broadcast sides -> no broadcast
+* Expected Res : Success
+* Date : 09/22/2017
+*/
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type TT as closed {
+ c_key : integer,
+ c_1 : integer,
+ c_2 : integer,
+ c_3 : integer,
+ c_4 : integer
+};
+
+create dataset t1(TT) primary key c_key;
+
+create dataset t2(TT) primary key c_key;
+
+select t1.c_4 as c0, t2.c_4 as c1
+from t1 join t2
+on t1.c_1 /*+ bcast */ = t2.c_1 and t1.c_2 /*+ bcast */ = t2.c_2 and t2.c_3 /*+ bcast */ = t1.c_3
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan
new file mode 100644
index 0000000..d45ccfe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_1.plan
@@ -0,0 +1,23 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$21, $$23, $$25][$$22, $$24, $$26] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan
new file mode 100644
index 0000000..d45ccfe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_2.plan
@@ -0,0 +1,23 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$21, $$23, $$25][$$22, $$24, $$26] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan
new file mode 100644
index 0000000..b7669dd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/hints/broadcast_hint_3.plan
@@ -0,0 +1,23 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$21, $$24, $$25][$$22, $$23, $$26] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$21, $$24, $$25] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$22, $$23, $$26] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/88b154df/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
index d66e6fc..652f062 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -178,33 +178,48 @@ public class JoinUtils {
return null;
}
AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
- IExpressionAnnotation ann = fexp.getAnnotations().get(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY);
- if (ann == null) {
- return null;
- }
- BroadcastSide side = (BroadcastSide) ann.getObject();
- if (side == null) {
- return null;
- }
- int i;
- switch (side) {
- case LEFT:
- i = 0;
- break;
- case RIGHT:
- i = 1;
- break;
- default:
- return null;
- }
- ArrayList<LogicalVariable> vars = new ArrayList<>();
- fexp.getArguments().get(i).getValue().getUsedVariables(vars);
- if (varsLeft.containsAll(vars)) {
- return BroadcastSide.LEFT;
- } else if (varsRight.containsAll(vars)) {
- return BroadcastSide.RIGHT;
+ FunctionIdentifier fi = fexp.getFunctionIdentifier();
+ if (fi.equals(AlgebricksBuiltinFunctions.AND)) {
+ BroadcastSide fBcastSide = null;
+ for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
+ BroadcastSide aBcastSide = getBroadcastJoinSide(a.getValue(), varsLeft, varsRight);
+ if (fBcastSide == null) {
+ fBcastSide = aBcastSide;
+ } else if (aBcastSide != null && !aBcastSide.equals(fBcastSide)) {
+ return null;
+ }
+ }
+ return fBcastSide;
} else {
- return null;
+ IExpressionAnnotation ann =
+ fexp.getAnnotations().get(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY);
+ if (ann == null) {
+ return null;
+ }
+ BroadcastSide side = (BroadcastSide) ann.getObject();
+ if (side == null) {
+ return null;
+ }
+ int i;
+ switch (side) {
+ case LEFT:
+ i = 0;
+ break;
+ case RIGHT:
+ i = 1;
+ break;
+ default:
+ return null;
+ }
+ ArrayList<LogicalVariable> vars = new ArrayList<>();
+ fexp.getArguments().get(i).getValue().getUsedVariables(vars);
+ if (varsLeft.containsAll(vars)) {
+ return BroadcastSide.LEFT;
+ } else if (varsRight.containsAll(vars)) {
+ return BroadcastSide.RIGHT;
+ } else {
+ return null;
+ }
}
}
}