You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/08/25 18:44:33 UTC
[45/51] [partial] incubator-asterixdb git commit: Change folder
structure for Java repackage
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/LoadRecordFieldsRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/LoadRecordFieldsRule.java
deleted file mode 100644
index 54e470c..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/LoadRecordFieldsRule.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.algebra.base.AsterixOperatorAnnotations;
-import edu.uci.ics.asterix.common.exceptions.AsterixRuntimeException;
-import edu.uci.ics.asterix.om.base.AString;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.optimizer.base.AnalysisUtil;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
-
- private ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
- if (context.checkIfInDontApplySet(this, op1)) {
- return false;
- }
-
- if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- AssignOperator a1 = (AssignOperator) op1;
- ILogicalExpression expr = getFirstExpr(a1);
- if (AnalysisUtil.isAccessToFieldRecord(expr)) {
- boolean res = findAndEliminateRedundantFieldAccess(a1);
- context.addToDontApplySet(this, op1);
- return res;
- }
- }
- exprVisitor.setTopOp(op1);
- exprVisitor.setContext(context);
- boolean res = op1.acceptExpressionTransform(exprVisitor);
- if (!res) {
- context.addToDontApplySet(this, op1);
- }
- if (res && op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
- // checking if we can annotate a Selection as using just one field
- // access
- SelectOperator sigma = (SelectOperator) op1;
- LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
- VariableUtilities.getUsedVariables(sigma, vars);
- if (vars.size() == 1) {
- // we can annotate Selection
- AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
- AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
- if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
- AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
- // f should be a call to a field/data access kind of
- // function
- sigma.getAnnotations().put(AsterixOperatorAnnotations.FIELD_ACCESS, f.getArguments().get(0));
- }
- }
- }
-
- // TODO: avoid having to recompute type env. here
- if (res) {
- OperatorPropertiesUtil.typeOpRec(opRef, context);
- }
- return res;
- }
-
- private static boolean pushFieldLoads(Mutable<ILogicalExpression> exprRef, AbstractLogicalOperator topOp,
- IOptimizationContext context) throws AlgebricksException {
- ILogicalExpression expr = exprRef.getValue();
- if (expr == null) {
- return false;
- }
- switch (expr.getExpressionTag()) {
- case FUNCTION_CALL: {
- AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
- FunctionIdentifier fi = f.getFunctionIdentifier();
- if (AlgebricksBuiltinFunctions.isComparisonFunction(fi)) {
- boolean b1 = pushFieldLoads(f.getArguments().get(0), topOp, context);
- boolean b2 = pushFieldLoads(f.getArguments().get(1), topOp, context);
- return b1 || b2;
- }
- if (fi.equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
- if (AnalysisUtil.numberOfVarsInExpr(f) == 0) {
- return false;
- }
- // create an assign
- LogicalVariable v = context.newVar();
- AssignOperator a2 = new AssignOperator(v, new MutableObject<ILogicalExpression>(f));
- pushFieldAssign(a2, topOp, context);
- context.computeAndSetTypeEnvironmentForOperator(a2);
- ILogicalExpression arg = f.getArguments().get(0).getValue();
- if (arg.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- VariableReferenceExpression ref = (VariableReferenceExpression) arg;
- LogicalVariable var = ref.getVariableReference();
- List<LogicalVariable> keys = context.findPrimaryKey(var);
- if (keys != null) {
- List<LogicalVariable> tail = new ArrayList<LogicalVariable>();
- tail.add(v);
- FunctionalDependency pk = new FunctionalDependency(keys, tail);
- context.addPrimaryKey(pk);
- }
- }
- exprRef.setValue(new VariableReferenceExpression(v));
- return true;
- } else {
- boolean pushed = false;
- for (Mutable<ILogicalExpression> argRef : f.getArguments()) {
- if (pushFieldLoads(argRef, topOp, context)) {
- pushed = true;
- }
- }
- return pushed;
- }
- }
- case CONSTANT:
- case VARIABLE: {
- return false;
- }
- default: {
- assert false;
- throw new IllegalArgumentException();
- }
- }
- }
-
- private static void pushFieldAssign(AssignOperator a2, AbstractLogicalOperator topOp, IOptimizationContext context)
- throws AlgebricksException {
- if (topOp.getInputs().size() == 1 && !topOp.hasNestedPlans()) {
- Mutable<ILogicalOperator> topChild = topOp.getInputs().get(0);
- // plugAccessAboveOp(a2, topChild, context);
- List<Mutable<ILogicalOperator>> a2InptList = a2.getInputs();
- a2InptList.clear();
- a2InptList.add(topChild);
- // and link it as child in the op. tree
- topOp.getInputs().set(0, new MutableObject<ILogicalOperator>(a2));
- findAndEliminateRedundantFieldAccess(a2);
- } else { // e.g., a join
- LinkedList<LogicalVariable> usedInAccess = new LinkedList<LogicalVariable>();
- VariableUtilities.getUsedVariables(a2, usedInAccess);
-
- LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
- VariableUtilities.getProducedVariables(topOp, produced2);
-
- if (OperatorPropertiesUtil.disjoint(produced2, usedInAccess)) {
- for (Mutable<ILogicalOperator> inp : topOp.getInputs()) {
- HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
- VariableUtilities.getLiveVariables(inp.getValue(), v2);
- if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
- pushAccessAboveOpRef(a2, inp, context);
- return;
- }
- }
- if (topOp.hasNestedPlans()) {
- AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) topOp;
- for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
- for (Mutable<ILogicalOperator> root : plan.getRoots()) {
- HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
- VariableUtilities.getLiveVariables(root.getValue(), v2);
- if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
- pushAccessAboveOpRef(a2, root, context);
- return;
- }
- }
- }
- }
- throw new AsterixRuntimeException("Field access " + getFirstExpr(a2)
- + " does not correspond to any input of operator " + topOp);
- }
- }
- }
-
- /**
- * Pushes one field-access assignment above toPushThroughChildRef
- *
- * @param toPush
- * @param toPushThroughChildRef
- */
- private static void pushAccessAboveOpRef(AssignOperator toPush, Mutable<ILogicalOperator> toPushThroughChildRef,
- IOptimizationContext context) throws AlgebricksException {
- List<Mutable<ILogicalOperator>> tpInpList = toPush.getInputs();
- tpInpList.clear();
- tpInpList.add(new MutableObject<ILogicalOperator>(toPushThroughChildRef.getValue()));
- toPushThroughChildRef.setValue(toPush);
- findAndEliminateRedundantFieldAccess(toPush);
- }
-
- /**
- * Rewrite
- * assign $x := field-access($y, "field")
- * assign $y := record-constructor { "field": Expr, ... }
- * into
- * assign $x := Expr
- * assign $y := record-constructor { "field": Expr, ... }
- *
- * @param toPush
- */
- private static boolean findAndEliminateRedundantFieldAccess(AssignOperator assign) throws AlgebricksException {
- ILogicalExpression expr = getFirstExpr(assign);
- AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
- ILogicalExpression arg0 = f.getArguments().get(0).getValue();
- if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
- return false;
- }
- VariableReferenceExpression vre = (VariableReferenceExpression) arg0;
- LogicalVariable recordVar = vre.getVariableReference();
- ILogicalExpression arg1 = f.getArguments().get(1).getValue();
- if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- return false;
- }
- ConstantExpression ce = (ConstantExpression) arg1;
- if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
- String fldName = ((AString) ((AsterixConstantValue) ce.getValue()).getObject()).getStringValue();
- ILogicalExpression fldExpr = findFieldExpression(assign, recordVar, fldName);
-
- if (fldExpr != null) {
- // check the liveness of the new expression
- List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
- fldExpr.getUsedVariables(usedVariables);
- List<LogicalVariable> liveInputVars = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(assign, liveInputVars);
- usedVariables.removeAll(liveInputVars);
- if (usedVariables.size() == 0) {
- assign.getExpressions().get(0).setValue(fldExpr);
- return true;
- } else {
- return false;
- }
- } else {
- return false;
- }
- } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
- // int fldIdx = ((IntegerLiteral) ce.getValue()).getValue();
- // TODO
- return false;
- } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED)) {
- return false;
- } else {
- throw new IllegalStateException();
- }
- }
-
- private static ILogicalExpression findFieldExpression(AbstractLogicalOperator op, LogicalVariable recordVar,
- String fldName) {
- for (Mutable<ILogicalOperator> child : op.getInputs()) {
- AbstractLogicalOperator opChild = (AbstractLogicalOperator) child.getValue();
- if (opChild.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- AssignOperator op2 = (AssignOperator) opChild;
- int i = 0;
- for (LogicalVariable var : op2.getVariables()) {
- if (var == recordVar) {
- AbstractLogicalExpression constr = (AbstractLogicalExpression) op2.getExpressions().get(i)
- .getValue();
- if (constr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return null;
- }
- AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) constr;
- if (!fce.getFunctionIdentifier().equals(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)
- && !fce.getFunctionIdentifier().equals(
- AsterixBuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR)) {
- return null;
- }
- Iterator<Mutable<ILogicalExpression>> fldIter = fce.getArguments().iterator();
- while (fldIter.hasNext()) {
- ILogicalExpression fldExpr = fldIter.next().getValue();
- if (fldExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
- ConstantExpression ce = (ConstantExpression) fldExpr;
- String f2 = ((AString) ((AsterixConstantValue) ce.getValue()).getObject())
- .getStringValue();
- if (fldName.equals(f2)) {
- return fldIter.next().getValue();
- }
- }
- fldIter.next();
- }
- return null;
- }
- i++;
- }
- } else if (opChild.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
- NestedTupleSourceOperator nts = (NestedTupleSourceOperator) opChild;
- AbstractLogicalOperator opBelowNestedPlan = (AbstractLogicalOperator) nts.getDataSourceReference()
- .getValue().getInputs().get(0).getValue();
- ILogicalExpression expr1 = findFieldExpression(opBelowNestedPlan, recordVar, fldName);
- if (expr1 != null) {
- return expr1;
- }
- }
- ILogicalExpression expr2 = findFieldExpression(opChild, recordVar, fldName);
- if (expr2 != null) {
- return expr2;
- }
- }
- return null;
- }
-
- private final class ExtractFieldLoadExpressionVisitor implements ILogicalExpressionReferenceTransform {
-
- private AbstractLogicalOperator topOp;
- private IOptimizationContext context;
-
- public void setTopOp(AbstractLogicalOperator topOp) {
- this.topOp = topOp;
- }
-
- public void setContext(IOptimizationContext context) {
- this.context = context;
- }
-
- @Override
- public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
- return pushFieldLoads(exprRef, topOp, context);
- }
-
- }
-
- private static ILogicalExpression getFirstExpr(AssignOperator assign) {
- return assign.getExpressions().get(0).getValue();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/NestGroupByRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/NestGroupByRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/NestGroupByRule.java
deleted file mode 100644
index 3ae35bc..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/NestGroupByRule.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class NestGroupByRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
- if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
- return false;
- }
- SubplanOperator subplan = (SubplanOperator) op1;
- if (subplan.getNestedPlans().size() != 1) {
- return false;
- }
- ILogicalPlan p = subplan.getNestedPlans().get(0);
- if (p.getRoots().size() != 1) {
- return false;
- }
-
- Set<LogicalVariable> free = new HashSet<LogicalVariable>();
- OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
- if (free.size() != 1) {
- return false;
- }
- LogicalVariable fVar = null;
- for (LogicalVariable v : free) {
- fVar = v;
- break;
- }
-
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
- if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
- return false;
- }
- GroupByOperator gby = (GroupByOperator) op2;
- if (gby.getNestedPlans().size() != 1) {
- return false;
- }
- ILogicalPlan p2 = gby.getNestedPlans().get(0);
- if (p2.getRoots().size() != 1) {
- return false;
- }
- Mutable<ILogicalOperator> r2 = p2.getRoots().get(0);
- AbstractLogicalOperator opr2 = (AbstractLogicalOperator) r2.getValue();
- if (opr2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- return false;
- }
- AggregateOperator aggOuter = (AggregateOperator) opr2;
- int posInAggList = aggOuter.getVariables().indexOf(fVar);
- if (posInAggList < 0) {
- return false;
- }
- AbstractLogicalOperator outerAggSon = (AbstractLogicalOperator) aggOuter.getInputs().get(0).getValue();
- if (outerAggSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
- return false;
- }
- ILogicalExpression eAgg = aggOuter.getExpressions().get(posInAggList).getValue();
- if (eAgg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return false;
- }
- AbstractFunctionCallExpression listifyCall = (AbstractFunctionCallExpression) eAgg;
- if (listifyCall.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
- return false;
- }
- ILogicalExpression argListify = listifyCall.getArguments().get(0).getValue();
- if (argListify.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
- return false;
- }
-
- Mutable<ILogicalOperator> r = p.getRoots().get(0);
- AbstractLogicalOperator opInS = (AbstractLogicalOperator) r.getValue();
- if (opInS.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- return false;
- }
- AggregateOperator aggInner = (AggregateOperator) opInS;
- do {
- opInS = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
- } while (opInS.getOperatorTag() == LogicalOperatorTag.ASSIGN);
- if (opInS.getOperatorTag() != LogicalOperatorTag.GROUP) {
- return false;
- }
- AbstractLogicalOperator unnestParent = opInS;
- AbstractLogicalOperator opUnder = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
- // skip Assigns
- while (opUnder.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- unnestParent = opUnder;
- opUnder = (AbstractLogicalOperator) opUnder.getInputs().get(0).getValue();
- }
- if (opUnder.getOperatorTag() != LogicalOperatorTag.UNNEST) {
- return false;
- }
- UnnestOperator unnest = (UnnestOperator) opUnder;
- AbstractLogicalOperator unnestSon = (AbstractLogicalOperator) unnest.getInputs().get(0).getValue();
- if (unnestSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
- return false;
- }
- NestedTupleSourceOperator innerNts = (NestedTupleSourceOperator) unnestSon;
-
- ILogicalExpression eUnnest = unnest.getExpressionRef().getValue();
- if (eUnnest.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return false;
- }
- AbstractFunctionCallExpression uf = (AbstractFunctionCallExpression) eUnnest;
- if (uf.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
- return false;
- }
- ILogicalExpression scanArg = uf.getArguments().get(0).getValue();
- if (scanArg.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
- return false;
- }
- if (((VariableReferenceExpression) scanArg).getVariableReference() != fVar) {
- return false;
- }
- LogicalVariable uVar = unnest.getVariable();
- GroupByOperator innerGby = (GroupByOperator) opInS;
- Set<LogicalVariable> freeInInnerGby = new HashSet<LogicalVariable>();
- OperatorPropertiesUtil.getFreeVariablesInSubplans(innerGby, freeInInnerGby);
- for (LogicalVariable v : freeInInnerGby) {
- if (v != uVar) {
- return false;
- }
- }
-
- unnestParent.getInputs().get(0).setValue(innerNts);
- LogicalVariable listifiedVar = ((VariableReferenceExpression) argListify).getVariableReference();
- substInSubplan(aggInner, uVar, listifiedVar, context);
- gby.getNestedPlans().add(p);
- innerNts.getDataSourceReference().setValue(gby);
- opRef.setValue(gby);
- OperatorPropertiesUtil.typePlan(p, context);
- OperatorPropertiesUtil.typePlan(p2, context);
- context.computeAndSetTypeEnvironmentForOperator(gby);
- return true;
-
- }
-
- private void substInSubplan(AggregateOperator aggInner, LogicalVariable v1, LogicalVariable v2,
- IOptimizationContext context) throws AlgebricksException {
- ILogicalOperator op = aggInner;
- while (op.getInputs().size() == 1) {
- VariableUtilities.substituteVariables(op, v1, v2, context);
- op = op.getInputs().get(0).getValue();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java
deleted file mode 100644
index 482a82b..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RunningAggregatePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnpartitionedPropertyComputer;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PullPositionalVariableFromUnnestRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
- return false;
- }
- UnnestOperator unnest = (UnnestOperator) op;
- LogicalVariable p = unnest.getPositionalVariable();
- if (p == null) {
- return false;
- }
- ArrayList<LogicalVariable> rOpVars = new ArrayList<LogicalVariable>();
- rOpVars.add(p);
- ArrayList<Mutable<ILogicalExpression>> rOpExprList = new ArrayList<Mutable<ILogicalExpression>>();
- StatefulFunctionCallExpression fce = new StatefulFunctionCallExpression(
- FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.TID), UnpartitionedPropertyComputer.INSTANCE);
- rOpExprList.add(new MutableObject<ILogicalExpression>(fce));
- RunningAggregateOperator rOp = new RunningAggregateOperator(rOpVars, rOpExprList);
- rOp.setExecutionMode(unnest.getExecutionMode());
- RunningAggregatePOperator rPop = new RunningAggregatePOperator();
- rOp.setPhysicalOperator(rPop);
- rOp.getInputs().add(new MutableObject<ILogicalOperator>(unnest));
- opRef.setValue(rOp);
- unnest.setPositionalVariable(null);
- context.computeAndSetTypeEnvironmentForOperator(rOp);
- context.computeAndSetTypeEnvironmentForOperator(unnest);
- return true;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java
deleted file mode 100644
index a29ebb7..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Pushes aggregate functions into a stand alone aggregate operator (no group by).
- */
-public class PushAggFuncIntoStandaloneAggregateRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- return false;
- }
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
- throws AlgebricksException {
- // Pattern to match: assign <-- aggregate <-- !(group-by)
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
- return false;
- }
- AssignOperator assignOp = (AssignOperator) op;
-
- Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
- AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
- if (op2.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
- AggregateOperator aggOp = (AggregateOperator) op2;
- // Make sure the agg expr is a listify.
- return pushAggregateFunction(aggOp, assignOp, context);
- } else if (op2.getOperatorTag() == LogicalOperatorTag.INNERJOIN
- || op2.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
- AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
- // Tries to push aggregates through the join.
- if (containsAggregate(assignOp.getExpressions()) && pushableThroughJoin(join)) {
- pushAggregateFunctionThroughJoin(join, assignOp, context);
- return true;
- }
- }
- return false;
- }
-
- /**
- * Recursively check whether the list of expressions contains an aggregate function.
- *
- * @param exprRefs
- * @return true if the list contains an aggregate function and false otherwise.
- */
- private boolean containsAggregate(List<Mutable<ILogicalExpression>> exprRefs) {
- for (Mutable<ILogicalExpression> exprRef : exprRefs) {
- ILogicalExpression expr = exprRef.getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- continue;
- }
- AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
- FunctionIdentifier funcIdent = AsterixBuiltinFunctions.getAggregateFunction(funcExpr
- .getFunctionIdentifier());
- if (funcIdent == null) {
- // Recursively look in func args.
- if (containsAggregate(funcExpr.getArguments())) {
- return true;
- }
- } else {
- // This is an aggregation function.
- return true;
- }
- }
- return false;
- }
-
- /**
- * Check whether the join is aggregate-pushable, that is,
- * 1) the join condition is true;
- * 2) each join branch produces only one tuple.
- *
- * @param join
- * @return true if pushable
- */
- private boolean pushableThroughJoin(AbstractBinaryJoinOperator join) {
- ILogicalExpression condition = join.getCondition().getValue();
- if (condition.equals(ConstantExpression.TRUE)) {
- // Checks if the aggregation functions are pushable through the join
- boolean pushable = true;
- for (Mutable<ILogicalOperator> branchRef : join.getInputs()) {
- AbstractLogicalOperator branch = (AbstractLogicalOperator) branchRef.getValue();
- if (branch.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
- pushable &= true;
- } else if (branch.getOperatorTag() == LogicalOperatorTag.INNERJOIN
- || branch.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
- AbstractBinaryJoinOperator childJoin = (AbstractBinaryJoinOperator) branch;
- pushable &= pushableThroughJoin(childJoin);
- } else {
- pushable &= false;
- }
- }
- return pushable;
- }
- return false;
- }
-
- /**
- * Does the actual push of aggregates for qualified joins.
- *
- * @param join
- * @param assignOp
- * that contains aggregate function calls.
- * @param context
- * @throws AlgebricksException
- */
- private void pushAggregateFunctionThroughJoin(AbstractBinaryJoinOperator join, AssignOperator assignOp,
- IOptimizationContext context) throws AlgebricksException {
- for (Mutable<ILogicalOperator> branchRef : join.getInputs()) {
- AbstractLogicalOperator branch = (AbstractLogicalOperator) branchRef.getValue();
- if (branch.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
- AggregateOperator aggOp = (AggregateOperator) branch;
- pushAggregateFunction(aggOp, assignOp, context);
- } else if (branch.getOperatorTag() == LogicalOperatorTag.INNERJOIN
- || branch.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
- AbstractBinaryJoinOperator childJoin = (AbstractBinaryJoinOperator) branch;
- pushAggregateFunctionThroughJoin(childJoin, assignOp, context);
- }
- }
- }
-
- private boolean pushAggregateFunction(AggregateOperator aggOp, AssignOperator assignOp, IOptimizationContext context)
- throws AlgebricksException {
- Mutable<ILogicalOperator> opRef3 = aggOp.getInputs().get(0);
- AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
- // If there's a group by below the agg, then we want to have the agg pushed into the group by.
- if (op3.getOperatorTag() == LogicalOperatorTag.GROUP) {
- return false;
- }
- if (aggOp.getVariables().size() != 1) {
- return false;
- }
- ILogicalExpression aggExpr = aggOp.getExpressions().get(0).getValue();
- if (aggExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return false;
- }
- AbstractFunctionCallExpression origAggFuncExpr = (AbstractFunctionCallExpression) aggExpr;
- if (origAggFuncExpr.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
- return false;
- }
-
- LogicalVariable aggVar = aggOp.getVariables().get(0);
- List<LogicalVariable> used = new LinkedList<LogicalVariable>();
- VariableUtilities.getUsedVariables(assignOp, used);
- if (!used.contains(aggVar)) {
- return false;
- }
-
- List<Mutable<ILogicalExpression>> srcAssignExprRefs = new LinkedList<Mutable<ILogicalExpression>>();
- if (fingAggFuncExprRef(assignOp.getExpressions(), aggVar, srcAssignExprRefs) == false) {
- return false;
- }
- if (srcAssignExprRefs.isEmpty()) {
- return false;
- }
-
- AbstractFunctionCallExpression aggOpExpr = (AbstractFunctionCallExpression) aggOp.getExpressions().get(0)
- .getValue();
- aggOp.getExpressions().clear();
- aggOp.getVariables().clear();
-
- for (Mutable<ILogicalExpression> srcAssignExprRef : srcAssignExprRefs) {
- AbstractFunctionCallExpression assignFuncExpr = (AbstractFunctionCallExpression) srcAssignExprRef
- .getValue();
- FunctionIdentifier aggFuncIdent = AsterixBuiltinFunctions.getAggregateFunction(assignFuncExpr
- .getFunctionIdentifier());
-
- // Push the agg func into the agg op.
-
- List<Mutable<ILogicalExpression>> aggArgs = new ArrayList<Mutable<ILogicalExpression>>();
- aggArgs.add(aggOpExpr.getArguments().get(0));
- AggregateFunctionCallExpression aggFuncExpr = AsterixBuiltinFunctions.makeAggregateFunctionExpression(
- aggFuncIdent, aggArgs);
- LogicalVariable newVar = context.newVar();
- aggOp.getVariables().add(newVar);
- aggOp.getExpressions().add(new MutableObject<ILogicalExpression>(aggFuncExpr));
-
- // The assign now just "renames" the variable to make sure the upstream plan still works.
- srcAssignExprRef.setValue(new VariableReferenceExpression(newVar));
- }
-
- context.computeAndSetTypeEnvironmentForOperator(aggOp);
- context.computeAndSetTypeEnvironmentForOperator(assignOp);
- return true;
- }
-
- private boolean fingAggFuncExprRef(List<Mutable<ILogicalExpression>> exprRefs, LogicalVariable aggVar,
- List<Mutable<ILogicalExpression>> srcAssignExprRefs) {
- for (Mutable<ILogicalExpression> exprRef : exprRefs) {
- ILogicalExpression expr = exprRef.getValue();
-
- if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- if (((VariableReferenceExpression) expr).getVariableReference().equals(aggVar)) {
- return false;
- }
- }
-
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- continue;
- }
- AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
- FunctionIdentifier funcIdent = AsterixBuiltinFunctions.getAggregateFunction(funcExpr
- .getFunctionIdentifier());
- if (funcIdent == null) {
- // Recursively look in func args.
- if (fingAggFuncExprRef(funcExpr.getArguments(), aggVar, srcAssignExprRefs) == false) {
- return false;
- }
-
- } else {
- // Check if this is the expr that uses aggVar.
- Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
- funcExpr.getUsedVariables(usedVars);
- if (usedVars.contains(aggVar)) {
- srcAssignExprRefs.add(exprRef);
- }
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java
deleted file mode 100644
index 90b2f25..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushAggregateIntoGroupbyRule implements IAlgebraicRewriteRule {
-
- @Override
- public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
- return false;
- }
-
- @Override
- public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
- Map<LogicalVariable, Integer> gbyAggVars = new HashMap<LogicalVariable, Integer>();
- Map<LogicalVariable, Integer> gbyAggVarToPlanIndex = new HashMap<LogicalVariable, Integer>();
- Map<LogicalVariable, GroupByOperator> gbyWithAgg = new HashMap<LogicalVariable, GroupByOperator>();
- Map<ILogicalExpression, ILogicalExpression> aggExprToVarExpr = new HashMap<ILogicalExpression, ILogicalExpression>();
- // first collect vars. referring to listified sequences
- boolean changed = collectVarsBottomUp(opRef, context, gbyAggVars, gbyWithAgg, gbyAggVarToPlanIndex,
- aggExprToVarExpr);
- if (changed) {
- removeRedundantListifies(opRef, context, gbyAggVars, gbyWithAgg, gbyAggVarToPlanIndex);
- }
- return changed;
- }
-
- private void removeRedundantListifies(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
- Map<LogicalVariable, Integer> gbyAggVars, Map<LogicalVariable, GroupByOperator> gbyWithAgg,
- Map<LogicalVariable, Integer> gbyAggVarToPlanIndex) throws AlgebricksException {
- for (LogicalVariable aggVar : gbyAggVars.keySet()) {
- int occurs = gbyAggVars.get(aggVar);
- if (occurs == 0) {
- GroupByOperator gbyOp = gbyWithAgg.get(aggVar);
- AggregateOperator aggOp = (AggregateOperator) gbyOp.getNestedPlans()
- .get(gbyAggVarToPlanIndex.get(aggVar)).getRoots().get(0).getValue();
- int pos = aggOp.getVariables().indexOf(aggVar);
- if (pos >= 0) {
- aggOp.getVariables().remove(pos);
- aggOp.getExpressions().remove(pos);
- List<LogicalVariable> producedVarsAtAgg = new ArrayList<LogicalVariable>();
- VariableUtilities.getProducedVariablesInDescendantsAndSelf(aggOp, producedVarsAtAgg);
- if (producedVarsAtAgg.isEmpty()) {
- gbyOp.getNestedPlans().remove(gbyAggVarToPlanIndex.get(aggVar));
- }
- }
- }
- }
- }
-
- private boolean collectVarsBottomUp(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
- Map<LogicalVariable, Integer> gbyListifyVarsCount, Map<LogicalVariable, GroupByOperator> gbyWithAgg,
- Map<LogicalVariable, Integer> gbyAggVarToPlanIndex,
- Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr) throws AlgebricksException {
- AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
- context.addToDontApplySet(this, op1);
- boolean change = false;
- for (Mutable<ILogicalOperator> child : op1.getInputs()) {
- if (collectVarsBottomUp(child, context, gbyListifyVarsCount, gbyWithAgg, gbyAggVarToPlanIndex,
- aggregateExprToVarExpr)) {
- change = true;
- }
- }
- // Need to use a list instead of a hash-set, because a var. may appear
- // several times in the same op.
- List<LogicalVariable> used = new LinkedList<LogicalVariable>();
- VariableUtilities.getUsedVariables(op1, used);
- switch (op1.getOperatorTag()) {
- case ASSIGN:
- case SELECT: {
- boolean found = false;
- // Do some prefiltering: check if the Assign uses any gby vars.
- for (LogicalVariable v : used) {
- if (gbyListifyVarsCount.get(v) != null) {
- found = true;
- break;
- }
- }
- if (found) {
- if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
- AssignOperator assign = (AssignOperator) op1;
- for (Mutable<ILogicalExpression> exprRef : assign.getExpressions()) {
- Pair<Boolean, ILogicalExpression> p = extractAggFunctionsFromExpression(exprRef,
- gbyWithAgg, aggregateExprToVarExpr, context);
- if (p.first) {
- change = true;
- exprRef.setValue(p.second);
- }
- }
- }
- if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
- SelectOperator select = (SelectOperator) op1;
- Mutable<ILogicalExpression> exprRef = select.getCondition();
- Pair<Boolean, ILogicalExpression> p = extractAggFunctionsFromExpression(exprRef, gbyWithAgg,
- aggregateExprToVarExpr, context);
- if (p.first) {
- change = true;
- exprRef.setValue(p.second);
- }
- }
- used.clear();
- VariableUtilities.getUsedVariables(op1, used);
- // increment the count for the ones which are still used
- for (LogicalVariable v : used) {
- Integer m = gbyListifyVarsCount.get(v);
- if (m != null) {
- gbyListifyVarsCount.put(v, m + 1);
- }
- }
- }
- break;
- }
- case SUBPLAN: {
- for (LogicalVariable v : used) {
- Integer m = gbyListifyVarsCount.get(v);
- if (m != null) {
- GroupByOperator gbyOp = gbyWithAgg.get(v);
- if (pushSubplanAsAggIntoGby(opRef, gbyOp, v, gbyListifyVarsCount, gbyWithAgg,
- gbyAggVarToPlanIndex, context)) {
- change = true;
- } else {
- gbyListifyVarsCount.put(v, m + 1);
- }
- break;
- }
- }
- break;
- }
- case GROUP: {
- List<LogicalVariable> vars = collectOneVarPerAggFromGroupOp((GroupByOperator) op1);
- if (vars != null) {
- for (int i = 0; i < vars.size(); i++) {
- LogicalVariable v = vars.get(i);
- if (v != null) {
- gbyListifyVarsCount.put(v, 0);
- gbyAggVarToPlanIndex.put(v, i);
- gbyWithAgg.put(v, (GroupByOperator) op1);
- }
- }
- }
- break;
- }
- default: {
- for (LogicalVariable v : used) {
- Integer m = gbyListifyVarsCount.get(v);
- if (m != null) {
- gbyListifyVarsCount.put(v, m + 1);
- }
- }
- }
- }
- return change;
- }
-
- private List<LogicalVariable> collectOneVarPerAggFromGroupOp(GroupByOperator group) {
- List<ILogicalPlan> nPlans = group.getNestedPlans();
- if (nPlans == null || nPlans.size() < 1) {
- return null;
- }
-
- List<LogicalVariable> aggVars = new ArrayList<LogicalVariable>();
- // test that the group-by computes a "listify" aggregate
- for (int i = 0; i < nPlans.size(); i++) {
- AbstractLogicalOperator topOp = (AbstractLogicalOperator) nPlans.get(i).getRoots().get(0).getValue();
- if (topOp.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- continue;
- }
- AggregateOperator agg = (AggregateOperator) topOp;
- if (agg.getVariables().size() != 1) {
- continue;
- }
- ILogicalExpression expr = agg.getExpressions().get(0).getValue();
- if (((AbstractLogicalExpression) expr).getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- continue;
- }
- AbstractFunctionCallExpression fceAgg = (AbstractFunctionCallExpression) expr;
- if (fceAgg.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
- continue;
- }
- aggVars.add(agg.getVariables().get(0));
- }
- return aggVars;
- }
-
- /**
- * @param expr
- * @param aggVars
- * @param gbyWithAgg
- * @param context
- * @return a pair whose first member is a boolean which is true iff
- * something was changed in the expression tree rooted at expr. The
- * second member is the result of transforming expr.
- * @throws AlgebricksException
- */
- private Pair<Boolean, ILogicalExpression> extractAggFunctionsFromExpression(Mutable<ILogicalExpression> exprRef,
- Map<LogicalVariable, GroupByOperator> gbyWithAgg,
- Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr, IOptimizationContext context)
- throws AlgebricksException {
- ILogicalExpression expr = exprRef.getValue();
- switch (expr.getExpressionTag()) {
- case FUNCTION_CALL: {
- AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
- FunctionIdentifier fi = AsterixBuiltinFunctions.getAggregateFunction(fce.getFunctionIdentifier());
- if (fi != null) {
- ILogicalExpression a1 = fce.getArguments().get(0).getValue();
- if (a1.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- LogicalVariable argVar = ((VariableReferenceExpression) a1).getVariableReference();
- GroupByOperator gbyOp = gbyWithAgg.get(argVar);
-
- if (gbyOp != null) {
- if (!aggregateExprToVarExpr.containsKey(expr)) {
- LogicalVariable newVar = context.newVar();
- AggregateFunctionCallExpression aggFun = AsterixBuiltinFunctions
- .makeAggregateFunctionExpression(fi, fce.getArguments());
- rewriteGroupByAggregate(argVar, gbyOp, aggFun, newVar, context);
- ILogicalExpression newVarExpr = new VariableReferenceExpression(newVar);
- aggregateExprToVarExpr.put(expr, newVarExpr);
- return new Pair<Boolean, ILogicalExpression>(Boolean.TRUE, newVarExpr);
- } else {
- ILogicalExpression varExpr = aggregateExprToVarExpr.get(expr);
- return new Pair<Boolean, ILogicalExpression>(Boolean.TRUE, varExpr);
- }
- }
- }
- }
-
- boolean change = false;
- for (Mutable<ILogicalExpression> a : fce.getArguments()) {
- Pair<Boolean, ILogicalExpression> aggArg = extractAggFunctionsFromExpression(a, gbyWithAgg,
- aggregateExprToVarExpr, context);
- if (aggArg.first.booleanValue()) {
- a.setValue(aggArg.second);
- change = true;
- }
- }
- return new Pair<Boolean, ILogicalExpression>(change, fce);
- }
- case VARIABLE:
- case CONSTANT: {
- return new Pair<Boolean, ILogicalExpression>(Boolean.FALSE, expr);
- }
- default: {
- throw new IllegalArgumentException();
- }
- }
- }
-
- private void rewriteGroupByAggregate(LogicalVariable oldAggVar, GroupByOperator gbyOp,
- AggregateFunctionCallExpression aggFun, LogicalVariable newAggVar, IOptimizationContext context)
- throws AlgebricksException {
- for (int j = 0; j < gbyOp.getNestedPlans().size(); j++) {
- AggregateOperator aggOp = (AggregateOperator) gbyOp.getNestedPlans().get(j).getRoots().get(0).getValue();
- int n = aggOp.getVariables().size();
- for (int i = 0; i < n; i++) {
- LogicalVariable v = aggOp.getVariables().get(i);
- if (v.equals(oldAggVar)) {
- AbstractFunctionCallExpression oldAggExpr = (AbstractFunctionCallExpression) aggOp.getExpressions()
- .get(i).getValue();
- AggregateFunctionCallExpression newAggFun = AsterixBuiltinFunctions
- .makeAggregateFunctionExpression(aggFun.getFunctionIdentifier(),
- new ArrayList<Mutable<ILogicalExpression>>());
- for (Mutable<ILogicalExpression> arg : oldAggExpr.getArguments()) {
- ILogicalExpression cloned = ((AbstractLogicalExpression) arg.getValue()).cloneExpression();
- newAggFun.getArguments().add(new MutableObject<ILogicalExpression>(cloned));
- }
- aggOp.getVariables().add(newAggVar);
- aggOp.getExpressions().add(new MutableObject<ILogicalExpression>(newAggFun));
- context.computeAndSetTypeEnvironmentForOperator(aggOp);
- break;
- }
- }
- }
- }
-
- private boolean pushSubplanAsAggIntoGby(Mutable<ILogicalOperator> subplanOpRef, GroupByOperator gbyOp,
- LogicalVariable varFromGroupAgg, Map<LogicalVariable, Integer> gbyAggVars,
- Map<LogicalVariable, GroupByOperator> gbyWithAgg, Map<LogicalVariable, Integer> gbyAggVarToPlanIndex,
- IOptimizationContext context) throws AlgebricksException {
- SubplanOperator subplan = (SubplanOperator) subplanOpRef.getValue();
- // only free var can be varFromGroupAgg
- HashSet<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
- OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
- for (LogicalVariable vFree : freeVars) {
- if (!vFree.equals(varFromGroupAgg)) {
- return false;
- }
- }
-
- List<ILogicalPlan> plans = subplan.getNestedPlans();
- if (plans.size() > 1) {
- return false;
- }
- ILogicalPlan p = plans.get(0);
- if (p.getRoots().size() > 1) {
- return false;
- }
- Mutable<ILogicalOperator> opRef = p.getRoots().get(0);
- AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
- return false;
- }
- AggregateOperator aggInSubplanOp = (AggregateOperator) op;
- LogicalVariable unnestVar = null;
- boolean pushableNestedSubplan = false;
- while (op.getInputs().size() == 1) {
- opRef = op.getInputs().get(0);
- op = (AbstractLogicalOperator) opRef.getValue();
- switch (op.getOperatorTag()) {
- case ASSIGN: {
- break;
- }
- case UNNEST: {
- UnnestOperator unnest = (UnnestOperator) op;
- if (unnest.getPositionalVariable() != null) {
- // TODO currently subplan with both accumulating and running aggregate is not supported.
- return false;
- }
- ILogicalExpression expr = unnest.getExpressionRef().getValue();
- if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
- return false;
- }
- AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) expr;
- if (fun.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
- return false;
- }
- ILogicalExpression arg0 = fun.getArguments().get(0).getValue();
- if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
- return false;
- }
- VariableReferenceExpression varExpr = (VariableReferenceExpression) arg0;
- if (!varExpr.getVariableReference().equals(varFromGroupAgg)) {
- return false;
- }
- opRef = op.getInputs().get(0);
- op = (AbstractLogicalOperator) opRef.getValue();
- if (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
- return false;
- }
- pushableNestedSubplan = true;
- unnestVar = unnest.getVariable();
- break;
- }
- default: {
- return false;
- }
- }
- }
- if (pushableNestedSubplan) {
- for (int i = 0; i < gbyOp.getNestedPlans().size(); i++) {
- Mutable<ILogicalOperator> gbyAggRef = gbyOp.getNestedPlans().get(i).getRoots().get(0);
- AggregateOperator gbyAgg = (AggregateOperator) gbyAggRef.getValue();
- Mutable<ILogicalOperator> gbyAggChildRef = gbyAgg.getInputs().get(0);
- OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar,
- findListifiedVariable(gbyAgg, varFromGroupAgg), true, context);
- gbyAgg.getVariables().addAll(aggInSubplanOp.getVariables());
- gbyAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
- for (LogicalVariable v : aggInSubplanOp.getVariables()) {
- gbyWithAgg.put(v, gbyOp);
- gbyAggVars.put(v, 0);
- gbyAggVarToPlanIndex.put(v, i);
- }
-
- Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0);
- if (opRef1InSubplan.getValue().getInputs().size() > 0) {
- Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
- AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
- if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
- List<Mutable<ILogicalOperator>> gbyInpList = gbyAgg.getInputs();
- gbyInpList.clear();
- gbyInpList.add(opRef1InSubplan);
- while (true) {
- opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
- op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
- if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) {
- List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs();
- opInpList.clear();
- opInpList.add(gbyAggChildRef);
- break;
- }
- opRef1InSubplan = opRef2InSubplan;
- if (opRef1InSubplan.getValue().getInputs().size() == 0) {
- throw new IllegalStateException("PushAggregateIntoGroupbyRule: could not find UNNEST.");
- }
- }
- }
- }
- subplanOpRef.setValue(subplan.getInputs().get(0).getValue());
- OperatorPropertiesUtil.typeOpRec(gbyAggRef, context);
- }
- return true;
- } else {
- return false;
- }
- }
-
- private LogicalVariable findListifiedVariable(AggregateOperator gbyAgg, LogicalVariable varFromGroupAgg) {
- int n = gbyAgg.getVariables().size();
-
- for (int i = 0; i < n; i++) {
- if (gbyAgg.getVariables().get(i).equals(varFromGroupAgg)) {
- AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) gbyAgg.getExpressions().get(i)
- .getValue();
- if (fce.getFunctionIdentifier().equals(AsterixBuiltinFunctions.LISTIFY)) {
- ILogicalExpression argExpr = fce.getArguments().get(0).getValue();
- if (((AbstractLogicalExpression) argExpr).getExpressionTag() == LogicalExpressionTag.VARIABLE) {
- return ((VariableReferenceExpression) argExpr).getVariableReference();
- }
- }
- }
- }
- return null;
- }
-
-}
\ No newline at end of file