You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2022/01/17 20:10:22 UTC
[asterixdb] branch master updated: [ASTERIXDB-3006][COMP] Change translation of inner joins in subplans
This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7a0f99b [ASTERIXDB-3006][COMP] Change translation of inner joins in subplans
new 1fd4cd3 Merge branch 'gerrit/neo'
7a0f99b is described below
commit 7a0f99b85cddf3ac1a157bcfe3ef1045683f3f90
Author: Dmitry Lychagin <dm...@couchbase.com>
AuthorDate: Fri Jan 14 18:49:50 2022 -0800
[ASTERIXDB-3006][COMP] Change translation of inner joins in subplans
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Translate inner join inside subplan into unnest/subplan
operators instead of a join operator. The join will be
introduced later by the optimizer
- This translation matches expectations of
InlineSubplanInputForNestedTupleSourceRule
- Add testcase
Change-Id: I46d6ca1bfaa9f76012adaa21a2aeca56b4d93afb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/14825
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
...InlineSubplanInputForNestedTupleSourceRule.java | 2 +-
.../SqlppExpressionToPlanTranslator.java | 25 +++++------
.../asterix/translator/TranslationContext.java | 7 ++++
.../queries/subquery/query-ASTERIXDB-3006.sqlpp | 38 +++++++++++++++++
.../results/subquery/query-ASTERIXDB-3006.plan | 49 ++++++++++++++++++++++
.../query-ASTERIXDB-3006.1.ddl.sqlpp | 30 +++++++++++++
.../query-ASTERIXDB-3006.2.update.sqlpp | 31 ++++++++++++++
.../query-ASTERIXDB-3006.3.query.sqlpp | 32 ++++++++++++++
.../query-ASTERIXDB-3006.3.adm | 2 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 +++
10 files changed, 208 insertions(+), 13 deletions(-)
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
index 9f932a8..6220d06 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineSubplanInputForNestedTupleSourceRule.java
@@ -638,9 +638,9 @@ public class InlineSubplanInputForNestedTupleSourceRule implements IAlgebraicRew
topJoinRef.setValue(ntsOp);
}
opRef.setValue(groupbyOp);
+ VariableUtilities.substituteVariables(groupbyOp, result.second, context);
OperatorManipulationUtil.computeTypeEnvironmentBottomUp(groupbyOp, context);
- VariableUtilities.substituteVariables(groupbyOp, result.second, context);
replacedVarMap.putAll(result.second);
return new Pair<>(true, replacedVarMap);
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index 65b0f03..1d226dd 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -345,7 +345,7 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
public Pair<ILogicalOperator, LogicalVariable> visit(JoinClause joinClause, Mutable<ILogicalOperator> leftInputRef)
throws CompilationException {
SourceLocation sourceLoc = joinClause.getSourceLocation();
- if (joinClause.getJoinType() == JoinType.INNER) {
+ if (joinClause.getJoinType() == JoinType.INNER && !context.inSubplan()) {
Mutable<ILogicalOperator> rightInputRef = uncorrelatedRightBranchStack.peek();
Pair<ILogicalOperator, LogicalVariable> rightBranch =
generateUnnestForBinaryCorrelateRightBranch(joinClause, rightInputRef, false, null);
@@ -362,7 +362,7 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
filter.getInputs().add(conditionExprOpPair.second);
filter.setSourceLocation(conditionExprOpPair.first.getSourceLocation());
return new Pair<>(filter, rightBranch.second);
- } else if (joinClause.getJoinType() == JoinType.LEFTOUTER) {
+ } else if (joinClause.getJoinType() == JoinType.INNER || joinClause.getJoinType() == JoinType.LEFTOUTER) {
// Creates a subplan operator.
SubplanOperator subplanOp = new SubplanOperator();
subplanOp.getInputs().add(leftInputRef);
@@ -442,21 +442,22 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
ILogicalPlan subplan = new ALogicalPlanImpl(new MutableObject<>(aggOp));
subplanOp.getNestedPlans().add(subplan);
- // Outer unnest the aggregated var from the subplan.
- LogicalVariable outerUnnestVar = context.newVar();
+ // Unnest the aggregated var from the subplan.
+ LogicalVariable unnestVar = context.newVar();
VariableReferenceExpression aggVarRefExpr = new VariableReferenceExpression(aggVar);
aggVarRefExpr.setSourceLocation(aggOp.getSourceLocation());
Pair<ILogicalExpression, Mutable<ILogicalOperator>> pUnnestExpr =
makeUnnestExpression(aggVarRefExpr, new MutableObject<>(subplanOp));
- LeftOuterUnnestOperator outerUnnestOp =
- new LeftOuterUnnestOperator(outerUnnestVar, new MutableObject<>(pUnnestExpr.first),
+ AbstractUnnestOperator unnestOp = joinClause.getJoinType() == JoinType.INNER
+ ? new UnnestOperator(unnestVar, new MutableObject<>(pUnnestExpr.first))
+ : new LeftOuterUnnestOperator(unnestVar, new MutableObject<>(pUnnestExpr.first),
translateLeftOuterMissingValue(joinClause.getOuterJoinMissingValueType()));
- outerUnnestOp.getInputs().add(pUnnestExpr.second);
- outerUnnestOp.setSourceLocation(aggOp.getSourceLocation());
- currentTopOp = outerUnnestOp;
+ unnestOp.getInputs().add(pUnnestExpr.second);
+ unnestOp.setSourceLocation(aggOp.getSourceLocation());
+ currentTopOp = unnestOp;
if (hasRightPosVar) {
- VariableReferenceExpression outerUnnestVarRef1 = new VariableReferenceExpression(outerUnnestVar);
+ VariableReferenceExpression outerUnnestVarRef1 = new VariableReferenceExpression(unnestVar);
outerUnnestVarRef1.setSourceLocation(joinClause.getRightVariable().getSourceLocation());
ScalarFunctionCallExpression fieldAccessForRightUnnestVar = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX),
@@ -464,7 +465,7 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
new MutableObject<>(new ConstantExpression(new AsterixConstantValue(new AInt32(0)))));
fieldAccessForRightUnnestVar.setSourceLocation(joinClause.getRightVariable().getSourceLocation());
- VariableReferenceExpression outerUnnestVarRef2 = new VariableReferenceExpression(outerUnnestVar);
+ VariableReferenceExpression outerUnnestVarRef2 = new VariableReferenceExpression(unnestVar);
outerUnnestVarRef2.setSourceLocation(joinClause.getPositionalVariable().getSourceLocation());
ScalarFunctionCallExpression fieldAccessForRightPosVar = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX),
@@ -496,7 +497,7 @@ public class SqlppExpressionToPlanTranslator extends LangExpressionToPlanTransla
assignOp.setSourceLocation(joinClause.getRightVariable().getSourceLocation());
currentTopOp = assignOp;
} else {
- context.setVar(joinClause.getRightVariable(), outerUnnestVar);
+ context.setVar(joinClause.getRightVariable(), unnestVar);
}
return new Pair<>(currentTopOp, null);
} else if (joinClause.getJoinType() == JoinType.RIGHTOUTER) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
index 33653ad..91629b6 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/TranslationContext.java
@@ -123,6 +123,13 @@ public final class TranslationContext implements IVariableContext {
}
}
+ /*
+ * Whether the translation is inside a subplan
+ */
+ public boolean inSubplan() {
+ return !stack.isEmpty();
+ }
+
/**
* @return the variables produced by the top operator in a subplan.
*/
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-3006.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-3006.sqlpp
new file mode 100644
index 0000000..6bf64c4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-3006.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3006
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create dataset ds1(id int not unknown) open type primary key id;
+create dataset ds2(id int not unknown) open type primary key id;
+
+select value ds1
+from ds1
+where exists(
+ select 1
+ from ds1.a[*].b b
+ join ds2 on ds2.x[0].y = b
+)
+order by ds1.id;
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-3006.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-3006.plan
new file mode 100644
index 0000000..a5264a9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-3006.plan
@@ -0,0 +1,49 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$64(ASC) ] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- SORT_GROUP_BY[$$78] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$48] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$48][$$61] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.ds1) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$61] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$b][$$51] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$b] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- UNNEST |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.ds1) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$51] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.ds2) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.1.ddl.sqlpp
new file mode 100644
index 0000000..bb5c8f5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.1.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3006
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create dataset ds1(id int not unknown) open type primary key id;
+create dataset ds2(id int not unknown) open type primary key id;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.2.update.sqlpp
new file mode 100644
index 0000000..fd8e76b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.2.update.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+insert into ds1 ([
+ { "id": 1, "a": [ { "b": 1 }, { "b": 11 }, { "b": 111 } ] },
+ { "id": 2, "a": [ { "b": 2 }, { "b": 22 }, { "b": 222 } ] },
+ { "id": 3, "a": [ { "b": 3 }, { "b": 33 }, { "b": 333 } ] }
+]);
+
+insert into ds2 ([
+ { "id": 101, "x": [ { "y": 33 }, { "y": 11 } ] },
+ { "id": 102, "x": [ { "y": 22 }, { "y": 11 } ] }
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.3.query.sqlpp
new file mode 100644
index 0000000..ad662ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.3.query.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test inner join inside a subquery
+ */
+use test;
+
+select value ds1
+from ds1
+where exists(
+ select 1
+ from ds1.a[*].b b
+ join ds2 on ds2.x[0].y = b
+)
+order by ds1.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.3.adm
new file mode 100644
index 0000000..99fe65f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-3006/query-ASTERIXDB-3006.3.adm
@@ -0,0 +1,2 @@
+{ "id": 2, "a": [ { "b": 2 }, { "b": 22 }, { "b": 222 } ] }
+{ "id": 3, "a": [ { "b": 3 }, { "b": 33 }, { "b": 333 } ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index c1aed2a..48b10ef 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -11374,6 +11374,11 @@
<output-dir compare="Text">query-ASTERIXDB-2845</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="subquery">
+ <compilation-unit name="query-ASTERIXDB-3006">
+ <output-dir compare="Text">query-ASTERIXDB-3006</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="subset-collection">
<test-case FilePath="subset-collection">