You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2022/11/08 18:36:08 UTC
Change in asterixdb[master]: PLEASE EDIT to provide a meaningful commit message!
From <mu...@gmail.com>:
murali4104@gmail.com has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17278 )
Change subject: PLEASE EDIT to provide a meaningful commit message!
......................................................................
PLEASE EDIT to provide a meaningful commit message!
The following commits from your working branch will be included:
commit 8c3d0e6f8ad65c1c7b25b610885a5268b51abd9d
Author: murali4104 <mu...@couchbase.com>
Date: Tue Nov 8 10:29:59 2022 -0800
back to JnI and JnJ
commit 755b2912e3a5cf95636a7322b6f06b082e8622bf
Author: murali4104 <mu...@couchbase.com>
Date: Tue Nov 8 10:06:29 2022 -0800
multiple nested loop changes by pusing assigns out of internal edges and not modifying the join predicate
commit 4f6bf520402c0b13d098c899ae925c7d30ec5672
Author: murali4104 <mu...@couchbase.com>
Date: Tue Nov 1 13:38:07 2022 -0700
added LOWEST COST printing
commit 46954eb73b240e673b5a86561fcfc92119321b10
Author: murali4104 <mu...@couchbase.com>
Date: Thu Oct 27 16:29:27 2022 -0700
selec. hint will not be overwritten
commit 44789c513655b6f0ab389c2681b18c9c2406d87f
Author: Ali Alsuliman <al...@gmail.com>
Date: Tue Sep 27 13:08:15 2022 -0700
WIP: join hints
Change-Id: Id9b147de07311a3cfd0c6b57887eb218bbcf4dac
---
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/PlanNode.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/cost/CostMethods.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
10 files changed, 455 insertions(+), 110 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/78/17278/1
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/cost/CostMethods.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/cost/CostMethods.java
index 5dafa54..caacf02 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/cost/CostMethods.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/cost/CostMethods.java
@@ -37,6 +37,7 @@
physOptConfig = context.getPhysicalOptimizationConfig();
blockSize = getBufferCachePageSize();
DOP = getDOP();
+ DOP = 1;
maxMemorySize = getMaxMemorySize();
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
index 4a37007..a6108bf 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
@@ -140,7 +140,12 @@
}
}
- printPlan(pp, (AbstractLogicalOperator) op, "Before calling new code. same plan still??");
+ if (internalEdges.size() > 0) {
+ pushAssignsIntoLeafInputs(joinLeafInputsHashMap, internalEdges);
+ }
+
+ printPlan(pp, (AbstractLogicalOperator) op, "Original Whole plan3");
+
int cheapestPlan = joinEnum.enumerateJoins();
printPlan(pp, (AbstractLogicalOperator) op, "After join enumeration. Must return same plan??");
if (cheapestPlan == PlanNode.NO_PLAN) {
@@ -150,9 +155,10 @@
PlanNode cheapestPlanNode = joinEnum.allPlans.get(cheapestPlan);
checkForMultipleUsesOfVariablesInJoinPreds(cheapestPlanNode, joinLeafInputsHashMap, context);
buildNewTree(cheapestPlanNode, joinLeafInputsHashMap, joinOps, new MutableInt(0));
+ printPlan(pp, (AbstractLogicalOperator) joinOps.get(0), "New Whole Plan after buildNewTree 1");
ILogicalOperator root = addConstantInternalEdgesAtTheTop(joinOps.get(0), internalEdges);
- printPlan(pp, (AbstractLogicalOperator) joinOps.get(0), "New Whole Plan after buildNewTree");
+ printPlan(pp, (AbstractLogicalOperator) joinOps.get(0), "New Whole Plan after buildNewTree 2");
printPlan(pp, (AbstractLogicalOperator) root, "New Whole Plan after buildNewTree");
// this will be the new root
@@ -254,6 +260,7 @@
// currently, will handle only assign statement and nothing else in an internal Edge.
// we can lift this restriction later if the need arises. This just makes some code easier.
internalEdges.add(nextOp);
+ //internalEdges.add(op); // better to store the parent; do this soon.
} else {
return false;
}
@@ -319,11 +326,10 @@
private void setAnnotation(AbstractFunctionCallExpression afcExpr, IExpressionAnnotation anno) {
FunctionIdentifier fi = afcExpr.getFunctionIdentifier();
List<Mutable<ILogicalExpression>> arguments = afcExpr.getArguments();
- int argumentCount = arguments.size();
if (fi.equals(AlgebricksBuiltinFunctions.AND)) {
- for (int i = 0; i < argumentCount; i++) {
- ILogicalExpression argument = arguments.get(i).getValue();
+ for (Mutable<ILogicalExpression> iLogicalExpressionMutable : arguments) {
+ ILogicalExpression argument = iLogicalExpressionMutable.getValue();
AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) argument;
expr.putAnnotation(anno);
}
@@ -332,6 +338,38 @@
}
}
+ //Internal edges are assign statements. The RHS has a variable in it.
+ // We need to find the internal edge that has a variable coming from this leaf leafInput.
+ private int findInternalEdge(ILogicalOperator leafInput, List<ILogicalOperator> internalEdges)
+ throws AlgebricksException {
+ int i = -1;
+
+ for (ILogicalOperator ie : internalEdges) {
+ i++;
+ // this will be an Assign, so no need to check
+ AssignOperator aOp = (AssignOperator) ie;
+ List<LogicalVariable> vars = new ArrayList<>();
+ aOp.getExpressions().get(0).getValue().getUsedVariables(vars);
+ HashSet<LogicalVariable> vars2 = new HashSet<>();
+ VariableUtilities.getLiveVariables(leafInput, vars2);
+ if (vars2.contains(vars.get(0))) {
+ return i;
+ }
+ i++;
+ }
+
+ return -1;
+ }
+
+ private ILogicalOperator addAssignToLeafInput(ILogicalOperator leafInput, ILogicalOperator internalEdge) {
+ ILogicalOperator root = leafInput;
+ // this will be an Assign, so no need to check
+ AssignOperator aOp = (AssignOperator) internalEdge;
+ aOp.getInputs().get(0).setValue(root);
+ root = aOp;
+ return root;
+ }
+
private void buildNewTree(PlanNode plan, HashMap<EmptyTupleSourceOperator, ILogicalOperator> joinLeafInputsHashMap,
List<ILogicalOperator> joinOps, MutableInt totalNumberOfJoins) {
// we have to move the inputs in op around so that they match the tree structure in pn
@@ -416,8 +454,11 @@
// in some very rare cases, there is an internal edge that has an assign statement such as $$var = 20 but this variable
// is not used anywhere in the current join graph but is used outside the current join graph. So we add this assign to the top of
// our plan, so the rest of the code will be happy. Strange that this assign appears in the join graph.
+ // Does this need a constant check??
private ILogicalOperator addConstantInternalEdgesAtTheTop(ILogicalOperator op,
List<ILogicalOperator> internalEdges) {
+ if (internalEdges.size() == 0)
+ return op;
ILogicalOperator root = op;
for (ILogicalOperator ie : internalEdges) {
// this will be an Assign, so no need to check
@@ -428,15 +469,16 @@
return root;
}
+ /*
public static void printPlan(IPlanPrettyPrinter pp, AbstractLogicalOperator op, String text)
throws AlgebricksException {
if (LOGGER.isTraceEnabled()) {
- pp.reset();
- pp.printOperator(op, true);
- LOGGER.trace("---------------------------- {}\n{}\n----------------------------", text, pp);
+ pp.reset();
+ pp.printOperator(op, true);
+ LOGGER.trace("---------------------------- {}\n{}\n----------------------------", text, pp);
}
}
-
+
private void printLeafPlans(IPlanPrettyPrinter pp,
HashMap<EmptyTupleSourceOperator, ILogicalOperator> joinLeafInputsHashMap) throws AlgebricksException {
Iterator<Map.Entry<EmptyTupleSourceOperator, ILogicalOperator>> li =
@@ -449,6 +491,30 @@
i++;
}
}
+
+ */
+
+ public static void printPlan(IPlanPrettyPrinter pp, AbstractLogicalOperator op, String text)
+ throws AlgebricksException {
+ System.out.println("---------------------------- " + text);
+ pp.reset();
+ pp.printOperator(op, true);
+ System.out.println(pp);
+ System.out.println("---------------------------- ");
+ }
+
+ protected void printLeafPlans(IPlanPrettyPrinter pp,
+ HashMap<EmptyTupleSourceOperator, ILogicalOperator> joinLeafInputsHashMap) throws AlgebricksException {
+ Iterator<Map.Entry<EmptyTupleSourceOperator, ILogicalOperator>> li =
+ joinLeafInputsHashMap.entrySet().iterator();
+ int i = 0;
+ while (li.hasNext()) {
+ Map.Entry<EmptyTupleSourceOperator, ILogicalOperator> pair = li.next();
+ ILogicalOperator element = pair.getValue();
+ printPlan(pp, (AbstractLogicalOperator) element, "Printing Leaf Input" + i);
+ i++;
+ }
+ }
private boolean allEqualityPreds(ILogicalExpression expr) {
List<Mutable<ILogicalExpression>> conjs = new ArrayList<>();
@@ -484,8 +550,7 @@
changes = false;
List<LogicalVariable> vars = new ArrayList<>();
exp.getUsedVariables(vars);
- Set<LogicalVariable> set = new LinkedHashSet<>();
- set.addAll(vars);
+ Set<LogicalVariable> set = new LinkedHashSet<>(vars);
if (set.size() < vars.size()) {
// walk thru vars and find the first instance of the duplicate
for (int i = 0; i < vars.size() - 1; i++) {
@@ -536,6 +601,26 @@
}
}
+ // for every internal edge assign (again assuming only 1 for now), find the corresponding leafInput place the assign
+ // on top of that LeafInput. Modify the joinLeafInputsHashMap as well.
+ private void pushAssignsIntoLeafInputs(HashMap<EmptyTupleSourceOperator, ILogicalOperator> joinLeafInputsHashMap,
+ List<ILogicalOperator> internalEdges) throws AlgebricksException {
+
+ for (Map.Entry<EmptyTupleSourceOperator, ILogicalOperator> mapElement : joinLeafInputsHashMap.entrySet()) {
+ ILogicalOperator joinLeafInput = mapElement.getValue();
+ EmptyTupleSourceOperator ets = mapElement.getKey();
+ HashSet<LogicalVariable> vars2 = new HashSet<>();
+ VariableUtilities.getLiveVariables(joinLeafInput, vars2);
+ int internalEdgeNumber = findInternalEdge(joinLeafInput, internalEdges);
+ if (internalEdgeNumber != -1) {
+ joinLeafInput = addAssignToLeafInput(joinLeafInput, internalEdges.get(internalEdgeNumber));
+ joinLeafInputsHashMap.put(ets, joinLeafInput);
+ internalEdges.remove(internalEdgeNumber); // no longer needed
+ }
+ }
+
+ }
+
private boolean substituteVarOnce(ILogicalExpression exp, LogicalVariable oldVar, LogicalVariable newVar) {
switch (exp.getExpressionTag()) {
case FUNCTION_CALL:
@@ -563,9 +648,9 @@
private boolean doAllDataSourcesHaveSamples(
List<Pair<EmptyTupleSourceOperator, DataSourceScanOperator>> emptyTupleAndDataSourceOps,
IOptimizationContext context) throws AlgebricksException {
- for (int i = 0; i < emptyTupleAndDataSourceOps.size(); i++) {
- if (emptyTupleAndDataSourceOps.get(i).getSecond() != null) {
- DataSourceScanOperator scanOp = emptyTupleAndDataSourceOps.get(i).getSecond();
+ for (Pair<EmptyTupleSourceOperator, DataSourceScanOperator> emptyTupleAndDataSourceOp : emptyTupleAndDataSourceOps) {
+ if (emptyTupleAndDataSourceOp.getSecond() != null) {
+ DataSourceScanOperator scanOp = emptyTupleAndDataSourceOp.getSecond();
Index index = joinEnum.getStatsHandle().findSampleIndex(scanOp, context);
if (index == null) {
return false;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
index b1a6fb6..31ee433 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinEnum.java
@@ -174,8 +174,8 @@
public ILogicalOperator findLeafInput(List<LogicalVariable> logicalVars) throws AlgebricksException {
Set<LogicalVariable> vars = new HashSet<>();
- for (int pos = 0; pos < emptyTupleAndDataSourceOps.size(); pos++) {
- EmptyTupleSourceOperator emptyOp = emptyTupleAndDataSourceOps.get(pos).getFirst();
+ for (Pair<EmptyTupleSourceOperator, DataSourceScanOperator> emptyTupleAndDataSourceOp : emptyTupleAndDataSourceOps) {
+ EmptyTupleSourceOperator emptyOp = emptyTupleAndDataSourceOp.getFirst();
ILogicalOperator op = joinLeafInputsHashMap.get(emptyOp);
vars.clear();
// this is expensive to do. So store this once and reuse
@@ -246,7 +246,7 @@
return eqPredFound ? andExpr : null;
}
- public HashJoinExpressionAnnotation.BuildSide findHashJoinHint(List<Integer> newJoinConditions) {
+ public HashJoinExpressionAnnotation findHashJoinHint(List<Integer> newJoinConditions) {
for (int i : newJoinConditions) {
JoinCondition jc = joinConditions.get(i);
if (jc.comparisonType != JoinCondition.comparisonOp.OP_EQ) {
@@ -257,14 +257,14 @@
AbstractFunctionCallExpression AFCexpr = (AbstractFunctionCallExpression) expr;
HashJoinExpressionAnnotation hjea = AFCexpr.getAnnotation(HashJoinExpressionAnnotation.class);
if (hjea != null) {
- return hjea.getBuildSide();
+ return hjea;
}
}
}
return null;
}
- public BroadcastExpressionAnnotation.BroadcastSide findBroadcastHashJoinHint(List<Integer> newJoinConditions) {
+ public BroadcastExpressionAnnotation findBroadcastHashJoinHint(List<Integer> newJoinConditions) {
for (int i : newJoinConditions) {
JoinCondition jc = joinConditions.get(i);
if (jc.comparisonType != JoinCondition.comparisonOp.OP_EQ) {
@@ -275,7 +275,7 @@
AbstractFunctionCallExpression AFCexpr = (AbstractFunctionCallExpression) expr;
BroadcastExpressionAnnotation bcasthjea = AFCexpr.getAnnotation(BroadcastExpressionAnnotation.class);
if (bcasthjea != null) {
- return bcasthjea.getBroadcastSide();
+ return bcasthjea;
}
}
}
@@ -302,7 +302,7 @@
for (int i = 1; i <= this.numberOfTerms; i++) {
if (name.equals(jnArray[i].datasetNames.get(0))) {
return i;
- } else if (name.equals(jnArray[i].alias)) {
+ } else if (name.equals(jnArray[i].aliases.get(0))) {
return i;
}
}
@@ -383,22 +383,23 @@
// now patch up any join conditions that have variables referenced in any internal assign statements.
List<LogicalVariable> usedVars = new ArrayList<>();
- for (JoinCondition jc : joinConditions) {
- usedVars.clear();
- ILogicalExpression expr = jc.joinCondition;
- expr.getUsedVariables(usedVars);
- for (ILogicalOperator ie : internalEdges) {
- AssignOperator aOp = (AssignOperator) ie;
- for (int i = 0; i < aOp.getVariables().size(); i++) {
- if (usedVars.contains(aOp.getVariables().get(i))) {
- OperatorManipulationUtil.replaceVarWithExpr((AbstractFunctionCallExpression) expr,
- aOp.getVariables().get(i), aOp.getExpressions().get(i).getValue());
- jc.joinCondition = expr;
- jc.selectivity = stats.getSelectivityFromAnnotationMain(jc.joinCondition, true);
+ if (false)
+ for (JoinCondition jc : joinConditions) {
+ usedVars.clear();
+ ILogicalExpression expr = jc.joinCondition;
+ expr.getUsedVariables(usedVars);
+ for (ILogicalOperator ie : internalEdges) {
+ AssignOperator aOp = (AssignOperator) ie;
+ for (int i = 0; i < aOp.getVariables().size(); i++) {
+ if (usedVars.contains(aOp.getVariables().get(i))) {
+ OperatorManipulationUtil.replaceVarWithExpr((AbstractFunctionCallExpression) expr,
+ aOp.getVariables().get(i), aOp.getExpressions().get(i).getValue());
+ jc.joinCondition = expr;
+ jc.selectivity = stats.getSelectivityFromAnnotationMain(jc.joinCondition, true);
+ }
}
}
}
- }
// now fill the datasetBits for each join condition.
for (JoinCondition jc : joinConditions) {
@@ -582,6 +583,10 @@
jn.datasetNames.addAll(jnI.datasetNames);
jn.datasetNames.addAll(jnJ.datasetNames);
Collections.sort(jn.datasetNames);
+ jn.aliases = new ArrayList<>();
+ jn.aliases.addAll(jnI.aliases);
+ jn.aliases.addAll(jnJ.aliases);
+ Collections.sort(jn.aliases);
jn.size = jnI.size + jnJ.size;
jn.cardinality = jn.computeJoinCardinality();
@@ -591,7 +596,8 @@
JoinNode jnIJ = jnArray[addPlansToThisJn];
jnIJ.jnArrayIndex = addPlansToThisJn;
- jnIJ.addMultiDatasetPlans(jnI, jnJ, level);
+ jnIJ.addMultiDatasetPlans(jnI, jnJ);
+ //jnIJ.addMultiDatasetPlans(i, j);
if (forceJoinOrderMode) {
break;
}
@@ -668,8 +674,8 @@
DataSourceScanOperator scanOp = emptyTupleAndDataSourceOps.get(i - 1).getSecond();
if (scanOp != null) {
DataSourceId id = (DataSourceId) scanOp.getDataSource().getId();
- jn.alias = findAlias(scanOp);
jn.datasetNames = new ArrayList<>(Collections.singleton(id.getDatasourceName()));
+ jn.aliases = new ArrayList<>(Collections.singleton(findAlias(scanOp)));
Index.SampleIndexDetails idxDetails;
Index index = stats.findSampleIndex(scanOp, optCtx);
if (index != null) {
@@ -695,6 +701,7 @@
} else {
// could be unnest or assign
jn.datasetNames = new ArrayList<>(Collections.singleton("unnestOrAssign"));
+ jn.aliases = new ArrayList<>(Collections.singleton("unnestOrAssign"));
jn.origCardinality = jn.cardinality = findInListCard(leafInput);
// just a guess
jn.size = 10;
@@ -726,24 +733,25 @@
}
IPlanPrettyPrinter pp = optCtx.getPrettyPrinter();
- if (LOGGER.isTraceEnabled()) {
- EnumerateJoinsRule.printPlan(pp, op, "Original Whole plan in JN 1");
- }
+ //if (LOGGER.isTraceEnabled()) {
+ EnumerateJoinsRule.printPlan(pp, op, "Original Whole plan in JN 1");
+ // }
findJoinConditions();
findIfJoinGraphIsConnected();
- if (LOGGER.isTraceEnabled()) {
- EnumerateJoinsRule.printPlan(pp, op, "Original Whole plan in JN 2");
- }
+ //if (LOGGER.isTraceEnabled()) {
+ EnumerateJoinsRule.printPlan(pp, op, "Original Whole plan in JN 2");
+ // }
markCompositeJoinPredicates();
int lastJnNum = enumerateHigherLevelJoinNodes();
JoinNode lastJn = jnArray[lastJnNum];
- if (LOGGER.isTraceEnabled()) {
- EnumerateJoinsRule.printPlan(pp, op, "Original Whole plan in JN END");
- LOGGER.trace(dumpJoinNodes(lastJnNum));
- }
+ System.out.println(dumpJoinNodes(lastJnNum));
+ //if (LOGGER.isTraceEnabled()) {
+ EnumerateJoinsRule.printPlan(pp, op, "Original Whole plan in JN END");
+ //LOGGER.trace(dumpJoinNodes(lastJnNum));
+ //}
// find the cheapest plan
int cheapestPlanIndex = lastJn.cheapestPlanIndex;
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
index cf4fcad..ed19205 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
@@ -68,7 +68,7 @@
protected int datasetBits; // this is bitmap of all the keyspaceBits present in this joinNode
protected List<Integer> datasetIndexes;
protected List<String> datasetNames;
- protected String alias;
+ protected List<String> aliases;
protected int cheapestPlanIndex;
protected ICost cheapestPlanCost;
protected double origCardinality; // without any selections
@@ -99,10 +99,7 @@
}
public boolean IsBaseLevelJoinNode() {
- if (this.jnArrayIndex <= joinEnum.numberOfTerms) {
- return true;
- }
- return false;
+ return this.jnArrayIndex <= joinEnum.numberOfTerms;
}
public boolean IsHigherLevelJoinNode() {
@@ -132,6 +129,7 @@
productJoinSels *= joinConditions.get(idx).selectivity;
}
}
+ System.out.println("JoinCard = " + productJoinCardinality * productJoinSels);
return productJoinCardinality * productJoinSels;
}
@@ -171,8 +169,8 @@
return rightJn;
}
- public String getAlias() {
- return alias;
+ public List<String> getAliases() {
+ return aliases;
}
public List<String> getDatasetNames() {
@@ -232,9 +230,9 @@
// Now call the rewritePre code
IntroduceJoinAccessMethodRule tmp = new IntroduceJoinAccessMethodRule();
- boolean temp = tmp.checkApplicable(new MutableObject<>(joinEnum.localJoinOp), joinEnum.optCtx);
- return temp;
+ boolean rt = tmp.checkApplicable(new MutableObject<>(joinEnum.localJoinOp), joinEnum.optCtx);
+ return rt;
}
/** one is a subset of two */
@@ -340,9 +338,7 @@
Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) throws AlgebricksException {
// Skip indexes with selectivity greater than 0.1, add the SKIP_SECONDARY_INDEX annotation to its expression.
double sel;
- Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
- while (amIt.hasNext()) {
- Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry = amIt.next();
+ for (Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry : analyzedAMs.entrySet()) {
AccessMethodAnalysisContext analysisCtx = amEntry.getValue();
Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt =
analysisCtx.getIteratorForIndexExprsAndVars();
@@ -381,7 +377,8 @@
}
}
- protected int buildHashJoinPlan(JoinNode leftJn, JoinNode rightJn, ILogicalExpression hashJoinExpr) {
+ protected int buildHashJoinPlan(JoinNode leftJn, JoinNode rightJn, ILogicalExpression hashJoinExpr,
+ HashJoinExpressionAnnotation hintHashJoin) {
List<PlanNode> allPlans = joinEnum.allPlans;
PlanNode pn;
ICost hjCost, childCosts, totalCost;
@@ -405,7 +402,8 @@
return PlanNode.NO_PLAN;
}
- if (rightJn.cardinality * rightJn.size <= leftJn.cardinality * leftJn.size || joinEnum.forceJoinOrderMode
+ if (rightJn.cardinality * rightJn.size <= leftJn.cardinality * leftJn.size || hintHashJoin != null
+ || joinEnum.forceJoinOrderMode
|| !joinEnum.queryPlanShape.equals(AlgebricksConfig.QUERY_PLAN_SHAPE_ZIGZAG)) {
// We want to build with the smaller side.
hjCost = joinEnum.getCostMethodsHandle().costHashJoin(this);
@@ -419,6 +417,9 @@
pn.planIndexes[0] = leftPlan;
pn.planIndexes[1] = rightPlan;
pn.joinOp = PlanNode.JoinMethod.HYBRID_HASH_JOIN; // need to check that all the conditions have equality predicates ONLY.
+ if (hintHashJoin != null) {
+ hintHashJoin.setBuildSide(HashJoinExpressionAnnotation.BuildSide.RIGHT);
+ }
pn.side = HashJoinExpressionAnnotation.BuildSide.RIGHT;
pn.joinExpr = hashJoinExpr;
pn.opCost = hjCost;
@@ -437,7 +438,8 @@
return PlanNode.NO_PLAN;
}
- protected int buildBroadcastHashJoinPlan(JoinNode leftJn, JoinNode rightJn, ILogicalExpression hashJoinExpr) {
+ protected int buildBroadcastHashJoinPlan(JoinNode leftJn, JoinNode rightJn, ILogicalExpression hashJoinExpr,
+ BroadcastExpressionAnnotation hintBroadcastHashJoin) {
List<PlanNode> allPlans = joinEnum.allPlans;
PlanNode pn;
ICost bcastHjCost, childCosts, totalCost;
@@ -461,7 +463,8 @@
return PlanNode.NO_PLAN;
}
- if (rightJn.cardinality * rightJn.size <= leftJn.cardinality * leftJn.size || joinEnum.forceJoinOrderMode
+ if (rightJn.cardinality * rightJn.size <= leftJn.cardinality * leftJn.size || hintBroadcastHashJoin != null
+ || joinEnum.forceJoinOrderMode
|| !joinEnum.queryPlanShape.equals(AlgebricksConfig.QUERY_PLAN_SHAPE_ZIGZAG)) {
// We want to broadcast and build with the smaller side.
bcastHjCost = joinEnum.getCostMethodsHandle().costBroadcastHashJoin(this);
@@ -475,6 +478,9 @@
pn.planIndexes[0] = leftPlan;
pn.planIndexes[1] = rightPlan;
pn.joinOp = PlanNode.JoinMethod.BROADCAST_HASH_JOIN; // need to check that all the conditions have equality predicates ONLY.
+ if (hintBroadcastHashJoin != null) {
+ hintBroadcastHashJoin.setBroadcastSide(BroadcastExpressionAnnotation.BroadcastSide.RIGHT);
+ }
pn.side = HashJoinExpressionAnnotation.BuildSide.RIGHT;
pn.joinExpr = hashJoinExpr;
pn.opCost = bcastHjCost;
@@ -559,11 +565,11 @@
cpJoinExpr = joinEnum.combineAllConditions(newJoinConditions);
} else if (hashJoinExpr != null && nestedLoopJoinExpr == null) {
cpJoinExpr = hashJoinExpr;
- } else if (hashJoinExpr == null && nestedLoopJoinExpr != null) {
+ } else if (hashJoinExpr == null) {
cpJoinExpr = nestedLoopJoinExpr;
- } else if (Objects.equals(hashJoinExpr, nestedLoopJoinExpr) == true) {
+ } else if (Objects.equals(hashJoinExpr, nestedLoopJoinExpr)) {
cpJoinExpr = hashJoinExpr;
- } else if (Objects.equals(hashJoinExpr, nestedLoopJoinExpr) == false) {
+ } else if (!Objects.equals(hashJoinExpr, nestedLoopJoinExpr)) {
ScalarFunctionCallExpression andExpr = new ScalarFunctionCallExpression(
BuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
andExpr.getArguments().add(new MutableObject<>(hashJoinExpr));
@@ -597,10 +603,9 @@
return PlanNode.NO_PLAN;
}
- protected Pair<Integer, ICost> addMultiDatasetPlans(JoinNode leftJn, JoinNode rightJn, int level)
- throws AlgebricksException {
- this.leftJn = leftJn;
- this.rightJn = rightJn;
+ protected Pair<Integer, ICost> addMultiDatasetPlans(JoinNode JnI, JoinNode JnJ) throws AlgebricksException {
+ this.leftJn = leftJn = JnI;
+ this.rightJn = rightJn = JnJ;
ICost noJoinCost = joinEnum.getCostHandle().maxCost();
if (leftJn.planIndexesArray.size() == 0 || rightJn.planIndexesArray.size() == 0) {
@@ -632,8 +637,8 @@
hjPlan = commutativeHjPlan = bcastHjPlan =
commutativeBcastHjPlan = nljPlan = commutativeNljPlan = cpPlan = commutativeCpPlan = PlanNode.NO_PLAN;
- HashJoinExpressionAnnotation.BuildSide hintHashJoin = joinEnum.findHashJoinHint(newJoinConditions);
- BroadcastExpressionAnnotation.BroadcastSide hintBroadcastHashJoin = null;
+ HashJoinExpressionAnnotation hintHashJoin = joinEnum.findHashJoinHint(newJoinConditions);
+ BroadcastExpressionAnnotation hintBroadcastHashJoin = null;
boolean hintNLJoin = false;
if (hintHashJoin == null) {
hintBroadcastHashJoin = joinEnum.findBroadcastHashJoinHint(newJoinConditions);
@@ -647,13 +652,21 @@
}
if (hintHashJoin != null) {
- hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr);
- if (!joinEnum.forceJoinOrderMode && hintHashJoin != HashJoinExpressionAnnotation.BuildSide.RIGHT) {
- commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ boolean build = (hintHashJoin.getBuildOrProbe() == HashJoinExpressionAnnotation.BuildOrProbe.BUILD);
+ String buildOrProbeObject = hintHashJoin.getName();
+ if (build
+ && (rightJn.datasetNames.contains(buildOrProbeObject)
+ || rightJn.aliases.contains(buildOrProbeObject))
+ || !build && (leftJn.datasetNames.contains(buildOrProbeObject)
+ || leftJn.aliases.contains(buildOrProbeObject))) {
+ hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr, hintHashJoin);
+ } else if (!joinEnum.forceJoinOrderMode) {
+ commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr, hintHashJoin);
}
if (hjPlan == PlanNode.NO_PLAN && commutativeHjPlan == PlanNode.NO_PLAN) {
// Hints are attached to predicates, so newJoinConditions should not be empty, but adding the check to be safe.
- if (!joinEnum.getJoinConditions().isEmpty() && !newJoinConditions.isEmpty()) {
+ if (!joinEnum.getJoinConditions().isEmpty() && !newJoinConditions.isEmpty()
+ && hintHashJoin.getBuildSide() == null) {
IWarningCollector warningCollector = joinEnum.optCtx.getWarningCollector();
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(
@@ -662,9 +675,9 @@
ErrorCode.INAPPLICABLE_HINT, "Hash join hint not applicable and was ignored"));
}
}
- bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr);
+ bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr, null);
if (!joinEnum.forceJoinOrderMode) {
- commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr, null);
}
nljPlan = buildNLJoinPlan(leftJn, rightJn, nestedLoopJoinExpr);
if (!joinEnum.forceJoinOrderMode) {
@@ -676,10 +689,12 @@
}
}
} else if (hintBroadcastHashJoin != null) {
- bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr);
- if (!joinEnum.forceJoinOrderMode
- && hintBroadcastHashJoin != BroadcastExpressionAnnotation.BroadcastSide.RIGHT) {
- commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ String broadcastObject = hintBroadcastHashJoin.getName();
+ if (rightJn.datasetNames.contains(broadcastObject) || rightJn.aliases.contains(broadcastObject)) {
+ bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr, hintBroadcastHashJoin);
+ } else if (!joinEnum.forceJoinOrderMode) {
+ commutativeBcastHjPlan =
+ buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr, hintBroadcastHashJoin);
}
if (bcastHjPlan == PlanNode.NO_PLAN && commutativeBcastHjPlan == PlanNode.NO_PLAN) {
// Hints are attached to predicates, so newJoinConditions should not be empty, but adding the check to be safe.
@@ -694,9 +709,9 @@
}
}
- hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr);
+ hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr, null);
if (!joinEnum.forceJoinOrderMode) {
- commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr, null);
}
nljPlan = buildNLJoinPlan(leftJn, rightJn, nestedLoopJoinExpr);
if (!joinEnum.forceJoinOrderMode) {
@@ -723,13 +738,13 @@
ErrorCode.INAPPLICABLE_HINT, "Index nested join hint not applicable and was ignored"));
}
}
- hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr);
+ hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr, null);
if (!joinEnum.forceJoinOrderMode) {
- commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr, null);
}
- bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr);
+ bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr, null);
if (!joinEnum.forceJoinOrderMode) {
- commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr, null);
}
cpPlan = buildCPJoinPlan(leftJn, rightJn, hashJoinExpr, nestedLoopJoinExpr);
if (!joinEnum.forceJoinOrderMode) {
@@ -737,13 +752,13 @@
}
}
} else {
- hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr);
+ hjPlan = buildHashJoinPlan(leftJn, rightJn, hashJoinExpr, null);
if (!joinEnum.forceJoinOrderMode) {
- commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ commutativeHjPlan = buildHashJoinPlan(rightJn, leftJn, hashJoinExpr, null);
}
- bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr);
+ bcastHjPlan = buildBroadcastHashJoinPlan(leftJn, rightJn, hashJoinExpr, null);
if (!joinEnum.forceJoinOrderMode) {
- commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr);
+ commutativeBcastHjPlan = buildBroadcastHashJoinPlan(rightJn, leftJn, hashJoinExpr, null);
}
nljPlan = buildNLJoinPlan(leftJn, rightJn, nestedLoopJoinExpr);
if (!joinEnum.forceJoinOrderMode) {
@@ -779,9 +794,9 @@
// This will avoid printing JoinNodes that have no plans
sb.append("Printing Join Node ").append(jnArrayIndex).append('\n');
sb.append("datasetNames ").append('\n');
- for (int j = 0; j < datasetNames.size(); j++) {
+ for (String datasetName : datasetNames) {
// Need to not print newline
- sb.append(datasetNames.get(j)).append(' ');
+ sb.append(datasetName).append(' ');
}
sb.append("datasetIndex ").append('\n');
for (int j = 0; j < datasetIndexes.size(); j++) {
@@ -799,7 +814,7 @@
int k = planIndexesArray.get(j);
PlanNode pn = allPlans.get(k);
sb.append("planIndexesArray [").append(j).append("] is ").append(k).append('\n');
- sb.append("Printing PlanNode ").append(k);
+ sb.append("Printing PlanNode ").append(k).append('\n');
if (IsBaseLevelJoinNode()) {
sb.append("DATA_SOURCE_SCAN").append('\n');
} else {
@@ -814,6 +829,7 @@
}
sb.append("card ").append((double) Math.round(cardinality * 100) / 100).append('\n');
sb.append("operator cost ").append(pn.opCost.computeTotalCost()).append('\n');
+ sb.append("------------------").append('\n');
sb.append("total cost ").append(pn.totalCost.computeTotalCost()).append('\n');
sb.append("jnIndexes ").append(pn.jnIndexes[0]).append(" ").append(pn.jnIndexes[1]).append('\n');
if (IsHigherLevelJoinNode()) {
@@ -839,9 +855,13 @@
public void printCostOfAllPlans(StringBuilder sb) {
List<PlanNode> allPlans = joinEnum.allPlans;
+ double minCost = 1.0e300;
for (int planIndex : planIndexesArray) {
- sb.append("plan ").append(planIndex).append(" cost is ")
- .append(allPlans.get(planIndex).totalCost.computeTotalCost()).append('\n');
+ double totalCost = allPlans.get(planIndex).totalCost.computeTotalCost();
+ sb.append("plan ").append(planIndex).append(" cost is ").append(totalCost).append('\n');
+ if (totalCost < minCost)
+ minCost = totalCost;
}
+ sb.append("LOWEST COST ").append(minCost).append('\n');
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/PlanNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/PlanNode.java
index 1dca15f..216673f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/PlanNode.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/PlanNode.java
@@ -67,6 +67,10 @@
jnIndexes = new int[2]; // join node index(es)
}
+ public int getIndex() {
+ return allPlansIndex;
+ }
+
public int[] getPlanIndexes() {
return planIndexes;
}
@@ -113,7 +117,7 @@
return getLeftPlanIndex() != NO_PLAN && getRightPlanIndex() != NO_PLAN;
}
- Pair<String, String> joinMethod() {
+ public Pair<String, String> joinMethod() {
if (this.joinOp == PlanNode.JoinMethod.HYBRID_HASH_JOIN) {
return new Pair<>("HASH JOIN", "HJ");
} else if (this.joinOp == PlanNode.JoinMethod.BROADCAST_HASH_JOIN) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
index 96bc412..28a79e6 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/Stats.java
@@ -148,16 +148,15 @@
}
}
- double s = 1.0;
+ double s;
PredicateCardinalityAnnotation pca = afcExpr.getAnnotation(PredicateCardinalityAnnotation.class);
if (pca != null) {
s = pca.getSelectivity();
- sel *= s;
} else {
JoinProductivityAnnotation jpa = afcExpr.getAnnotation(JoinProductivityAnnotation.class);
s = findJoinSelectivity(jpa, afcExpr);
- sel *= s;
}
+ sel *= s;
if (join && s == 1.0) {
// assume no selectivity was assigned
joinEnum.singleDatasetPreds.add(afcExpr);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 64c73af..26b665b 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -414,6 +414,32 @@
return new SQLPPParser(text).parseParenthesizedIdentifierList();
}
+ private Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> parseHashJoinParams() throws CompilationException {
+ return parseImpl(new ParseFunction<Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>>() {
+ @Override
+ public Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> parse() throws ParseException {
+ return SQLPPParser.this.buildOrProbeParenthesizedIdentifier();
+ }
+ });
+ }
+
+ private static Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> parseHashJoinParams(String text) throws CompilationException {
+ return new SQLPPParser(text).parseHashJoinParams();
+ }
+
+ private String parseBroadcastJoinParams() throws CompilationException {
+ return parseImpl(new ParseFunction<String>() {
+ @Override
+ public String parse() throws ParseException {
+ return SQLPPParser.this.parenthesizedIdentifier();
+ }
+ });
+ }
+
+ private static String parseBroadcastJoinParams(String text) throws CompilationException {
+ return new SQLPPParser(text).parseBroadcastJoinParams();
+ }
+
private List<Literal> parseParenthesizedLiteralList() throws CompilationException {
return parseImpl(new ParseFunction<List<Literal>>() {
@Override
@@ -692,9 +718,25 @@
// attach hint to global scope
return new JoinProductivityAnnotation (productivity, leftSideDataSet);
case HASH_BROADCAST_JOIN_HINT:
- return new BroadcastExpressionAnnotation(BroadcastExpressionAnnotation.BroadcastSide.RIGHT);
+ if (hintToken.hintParams == null) {
+ return new BroadcastExpressionAnnotation();
+ }
+ else {
+ // if parameter parsing fails then return hint annotation without parameters
+ onParseErrorReturn = new BroadcastExpressionAnnotation();
+ String name = parseBroadcastJoinParams(hintToken.hintParams);
+ return new BroadcastExpressionAnnotation(name);
+ }
case HASH_JOIN_HINT:
- return new HashJoinExpressionAnnotation(HashJoinExpressionAnnotation.BuildSide.RIGHT);
+ if (hintToken.hintParams == null) {
+ return new HashJoinExpressionAnnotation();
+ }
+ else {
+ // if parameter parsing fails then return hint annotation without parameters
+ onParseErrorReturn = new HashJoinExpressionAnnotation();
+ Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> pair = parseHashJoinParams(hintToken.hintParams);
+ return new HashJoinExpressionAnnotation(pair);
+ }
case INDEXED_NESTED_LOOP_JOIN_HINT:
if (hintToken.hintParams == null) {
return IndexedNLJoinExpressionAnnotation.INSTANCE_ANY_INDEX;
@@ -3163,6 +3205,40 @@
}
}
+Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> buildOrProbeParenthesizedIdentifier() throws ParseException:
+{
+ Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> pair = new Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>(HashJoinExpressionAnnotation.BuildOrProbe.BUILD, "");
+ String ident1 = null;
+ String ident2 = null;
+}
+{
+ ident1 = Identifier() <LEFTPAREN> ident2 = Identifier() <RIGHTPAREN>
+ {
+ // check
+ if (ident1.equals("build")) {
+ return new Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>(HashJoinExpressionAnnotation.BuildOrProbe.BUILD, ident2);
+ }
+ else if (ident1.equals("probe")) {
+ return new Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>(HashJoinExpressionAnnotation.BuildOrProbe.PROBE, ident2);
+ }
+ else {
+ return pair;
+ }
+ return pair;
+ }
+}
+
+String parenthesizedIdentifier() throws ParseException:
+{
+ String ident = null;
+}
+{
+ <LEFTPAREN> ident = Identifier() <RIGHTPAREN>
+ {
+ return ident;
+ }
+}
+
List<Literal> ParenthesizedLiteralList() throws ParseException:
{
List<Literal> list = new ArrayList<Literal>();
@@ -3591,6 +3667,10 @@
annotation = parseExpressionAnnotation(hintToken);
annotationList.add(annotation);
hintToken = hintToken.specialToken;
+ if (hintToken != null) {
+ SourceLocation sourceLoc = getSourceLocation(hintToken);
+ hintCollector.remove(sourceLoc);
+ }
}
String operator = opToken.image.toLowerCase();
if (operator.equals("<>")){
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
index 79b9e2c..343df8d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
@@ -25,7 +25,6 @@
public enum BroadcastSide {
LEFT,
RIGHT;
-
public static BroadcastSide getOppositeSide(BroadcastSide side) {
switch (side) {
case LEFT:
@@ -38,13 +37,33 @@
}
}
- private final BroadcastSide side;
+ private final String name;
+ private BroadcastSide side;
+
+ public BroadcastExpressionAnnotation() {
+ this.name = null;
+ this.side = null;
+ }
+
+ public BroadcastExpressionAnnotation(String name) {
+ this.name = name;
+ this.side = null;
+ }
public BroadcastExpressionAnnotation(BroadcastSide side) {
+ this.name = null;
this.side = Objects.requireNonNull(side);
}
+ public String getName() {
+ return name;
+ }
+
public BroadcastSide getBroadcastSide() {
return side;
}
+
+ public void setBroadcastSide(BroadcastSide side) {
+ this.side = side;
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
index 02ba2db..8b6ce7e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/HashJoinExpressionAnnotation.java
@@ -21,19 +21,55 @@
import java.util.Objects;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
public class HashJoinExpressionAnnotation implements IExpressionAnnotation {
+
+ public enum BuildOrProbe {
+ BUILD,
+ PROBE
+ }
+
public enum BuildSide {
LEFT,
RIGHT
}
- private final BuildSide side;
+ private final BuildOrProbe buildOrProbe;
+ private final String name;
+ private BuildSide side;
+
+ public HashJoinExpressionAnnotation() {
+ this.buildOrProbe = null;
+ this.name = null;
+ this.side = null;
+ }
+
+ public HashJoinExpressionAnnotation(Pair<BuildOrProbe, String> pair) {
+ this.buildOrProbe = pair.getFirst();
+ this.name = pair.getSecond();
+ this.side = null;
+ }
public HashJoinExpressionAnnotation(BuildSide side) {
+ this.buildOrProbe = null;
+ this.name = null;
this.side = Objects.requireNonNull(side);
}
+ public BuildOrProbe getBuildOrProbe() {
+ return buildOrProbe;
+ }
+
+ public String getName() {
+ return name;
+ }
+
public BuildSide getBuildSide() {
return side;
}
+
+ public void setBuildSide(BuildSide side) {
+ this.side = side;
+ }
}
\ No newline at end of file
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
index 1ac7614..acbb100 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -32,6 +32,8 @@
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
+import org.apache.hyracks.algebricks.core.algebra.expressions.HashJoinExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.expressions.HashJoinExpressionAnnotation.BuildSide;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
@@ -65,11 +67,35 @@
List<LogicalVariable> varsRight = op.getInputs().get(1).getValue().getSchema();
ILogicalExpression conditionExpr = op.getCondition().getValue();
if (isHashJoinCondition(conditionExpr, varsLeft, varsRight, sideLeft, sideRight)) {
- BroadcastSide side = getBroadcastJoinSide(conditionExpr);
- if (side == null) {
- setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+ BroadcastSide broadcastSide = getBroadcastJoinSide(conditionExpr);
+ if (broadcastSide == null) {
+ BuildSide buildSide = getHashJoinBuildSide(conditionExpr);
+ if (buildSide == null) {
+ setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+ } else {
+ switch (buildSide) {
+ case RIGHT:
+ setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+ break;
+ case LEFT:
+ if (op.getJoinKind() == AbstractBinaryJoinOperator.JoinKind.INNER) {
+ Mutable<ILogicalOperator> opRef0 = op.getInputs().get(0);
+ Mutable<ILogicalOperator> opRef1 = op.getInputs().get(1);
+ ILogicalOperator tmp = opRef0.getValue();
+ opRef0.setValue(opRef1.getValue());
+ opRef1.setValue(tmp);
+ setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideRight, sideLeft, context);
+ } else {
+ setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+ }
+ break;
+ default:
+ // This should never happen
+ throw new IllegalStateException(buildSide.toString());
+ }
+ }
} else {
- switch (side) {
+ switch (broadcastSide) {
case RIGHT:
setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideLeft, sideRight, context);
break;
@@ -87,7 +113,7 @@
break;
default:
// This should never happen
- throw new IllegalStateException(side.toString());
+ throw new IllegalStateException(broadcastSide.toString());
}
}
} else {
@@ -214,6 +240,32 @@
return null;
}
+ private static BuildSide getHashJoinBuildSide(ILogicalExpression e) {
+ BuildSide side = null;
+ if (e.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return null;
+ }
+ AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
+ FunctionIdentifier fi = fexp.getFunctionIdentifier();
+ if (fi.equals(AlgebricksBuiltinFunctions.AND)) {
+ for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
+ BuildSide newSide = getHashJoinBuildSide(a.getValue());
+ if (side == null) {
+ side = newSide;
+ } else if (newSide != null && !newSide.equals(side)) {
+ return null;
+ }
+ }
+ return side;
+ } else {
+ HashJoinExpressionAnnotation hashJoinAnnnotation = fexp.getAnnotation(HashJoinExpressionAnnotation.class);
+ if (hashJoinAnnnotation != null) {
+ return hashJoinAnnnotation.getBuildSide();
+ }
+ }
+ return null;
+ }
+
private static void warnIfCrossProduct(ILogicalExpression conditionExpr, SourceLocation sourceLoc,
IOptimizationContext context) {
if (OperatorPropertiesUtil.isAlwaysTrueCond(conditionExpr) && sourceLoc != null) {
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17278
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Id9b147de07311a3cfd0c6b57887eb218bbcf4dac
Gerrit-Change-Number: 17278
Gerrit-PatchSet: 1
Gerrit-Owner: murali4104@gmail.com
Gerrit-MessageType: newchange