[30/51] [partial] incubator-asterixdb git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java
new file mode 100644
index 0000000..482a82b
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PullPositionalVariableFromUnnestRule.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.aql.util.FunctionUtils;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RunningAggregatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnpartitionedPropertyComputer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PullPositionalVariableFromUnnestRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return false;
+        }
+        UnnestOperator unnest = (UnnestOperator) op;
+        LogicalVariable p = unnest.getPositionalVariable();
+        if (p == null) {
+            return false;
+        }
+        ArrayList<LogicalVariable> rOpVars = new ArrayList<LogicalVariable>();
+        rOpVars.add(p);
+        ArrayList<Mutable<ILogicalExpression>> rOpExprList = new ArrayList<Mutable<ILogicalExpression>>();
+        StatefulFunctionCallExpression fce = new StatefulFunctionCallExpression(
+                FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.TID), UnpartitionedPropertyComputer.INSTANCE);
+        rOpExprList.add(new MutableObject<ILogicalExpression>(fce));
+        RunningAggregateOperator rOp = new RunningAggregateOperator(rOpVars, rOpExprList);
+        rOp.setExecutionMode(unnest.getExecutionMode());
+        RunningAggregatePOperator rPop = new RunningAggregatePOperator();
+        rOp.setPhysicalOperator(rPop);
+        rOp.getInputs().add(new MutableObject<ILogicalOperator>(unnest));
+        opRef.setValue(rOp);
+        unnest.setPositionalVariable(null);
+        context.computeAndSetTypeEnvironmentForOperator(rOp);
+        context.computeAndSetTypeEnvironmentForOperator(unnest);
+        return true;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java
new file mode 100644
index 0000000..a29ebb7
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggFuncIntoStandaloneAggregateRule.java
@@ -0,0 +1,265 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Pushes aggregate functions into a stand alone aggregate operator (no group by).
+ */
+public class PushAggFuncIntoStandaloneAggregateRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        // Pattern to match: assign <-- aggregate <-- !(group-by)
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator assignOp = (AssignOperator) op;
+
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        if (op2.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+            AggregateOperator aggOp = (AggregateOperator) op2;
+            // Make sure the agg expr is a listify.
+            return pushAggregateFunction(aggOp, assignOp, context);
+        } else if (op2.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+                || op2.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+            AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+            // Tries to push aggregates through the join.
+            if (containsAggregate(assignOp.getExpressions()) && pushableThroughJoin(join)) {
+                pushAggregateFunctionThroughJoin(join, assignOp, context);
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Recursively check whether the list of expressions contains an aggregate function.
+     * 
+     * @param exprRefs
+     * @return true if the list contains an aggregate function and false otherwise.
+     */
+    private boolean containsAggregate(List<Mutable<ILogicalExpression>> exprRefs) {
+        for (Mutable<ILogicalExpression> exprRef : exprRefs) {
+            ILogicalExpression expr = exprRef.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+                continue;
+            }
+            AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+            FunctionIdentifier funcIdent = AsterixBuiltinFunctions.getAggregateFunction(funcExpr
+                    .getFunctionIdentifier());
+            if (funcIdent == null) {
+                // Recursively look in func args.
+                if (containsAggregate(funcExpr.getArguments())) {
+                    return true;
+                }
+            } else {
+                // This is an aggregation function.
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Check whether the join is aggregate-pushable, that is,
+     * 1) the join condition is true;
+     * 2) each join branch produces only one tuple.
+     * 
+     * @param join
+     * @return true if pushable
+     */
+    private boolean pushableThroughJoin(AbstractBinaryJoinOperator join) {
+        ILogicalExpression condition = join.getCondition().getValue();
+        if (condition.equals(ConstantExpression.TRUE)) {
+            // Checks if the aggregation functions are pushable through the join
+            boolean pushable = true;
+            for (Mutable<ILogicalOperator> branchRef : join.getInputs()) {
+                AbstractLogicalOperator branch = (AbstractLogicalOperator) branchRef.getValue();
+                if (branch.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+                    pushable &= true;
+                } else if (branch.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+                        || branch.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+                    AbstractBinaryJoinOperator childJoin = (AbstractBinaryJoinOperator) branch;
+                    pushable &= pushableThroughJoin(childJoin);
+                } else {
+                    pushable &= false;
+                }
+            }
+            return pushable;
+        }
+        return false;
+    }
+
+    /**
+     * Does the actual push of aggregates for qualified joins.
+     * 
+     * @param join
+     * @param assignOp
+     *            that contains aggregate function calls.
+     * @param context
+     * @throws AlgebricksException
+     */
+    private void pushAggregateFunctionThroughJoin(AbstractBinaryJoinOperator join, AssignOperator assignOp,
+            IOptimizationContext context) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> branchRef : join.getInputs()) {
+            AbstractLogicalOperator branch = (AbstractLogicalOperator) branchRef.getValue();
+            if (branch.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+                AggregateOperator aggOp = (AggregateOperator) branch;
+                pushAggregateFunction(aggOp, assignOp, context);
+            } else if (branch.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+                    || branch.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+                AbstractBinaryJoinOperator childJoin = (AbstractBinaryJoinOperator) branch;
+                pushAggregateFunctionThroughJoin(childJoin, assignOp, context);
+            }
+        }
+    }
+
+    private boolean pushAggregateFunction(AggregateOperator aggOp, AssignOperator assignOp, IOptimizationContext context)
+            throws AlgebricksException {
+        Mutable<ILogicalOperator> opRef3 = aggOp.getInputs().get(0);
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
+        // If there's a group by below the agg, then we want to have the agg pushed into the group by.
+        if (op3.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        if (aggOp.getVariables().size() != 1) {
+            return false;
+        }
+        ILogicalExpression aggExpr = aggOp.getExpressions().get(0).getValue();
+        if (aggExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression origAggFuncExpr = (AbstractFunctionCallExpression) aggExpr;
+        if (origAggFuncExpr.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
+            return false;
+        }
+
+        LogicalVariable aggVar = aggOp.getVariables().get(0);
+        List<LogicalVariable> used = new LinkedList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(assignOp, used);
+        if (!used.contains(aggVar)) {
+            return false;
+        }
+
+        List<Mutable<ILogicalExpression>> srcAssignExprRefs = new LinkedList<Mutable<ILogicalExpression>>();
+        if (fingAggFuncExprRef(assignOp.getExpressions(), aggVar, srcAssignExprRefs) == false) {
+            return false;
+        }
+        if (srcAssignExprRefs.isEmpty()) {
+            return false;
+        }
+
+        AbstractFunctionCallExpression aggOpExpr = (AbstractFunctionCallExpression) aggOp.getExpressions().get(0)
+                .getValue();
+        aggOp.getExpressions().clear();
+        aggOp.getVariables().clear();
+
+        for (Mutable<ILogicalExpression> srcAssignExprRef : srcAssignExprRefs) {
+            AbstractFunctionCallExpression assignFuncExpr = (AbstractFunctionCallExpression) srcAssignExprRef
+                    .getValue();
+            FunctionIdentifier aggFuncIdent = AsterixBuiltinFunctions.getAggregateFunction(assignFuncExpr
+                    .getFunctionIdentifier());
+
+            // Push the agg func into the agg op.                
+
+            List<Mutable<ILogicalExpression>> aggArgs = new ArrayList<Mutable<ILogicalExpression>>();
+            aggArgs.add(aggOpExpr.getArguments().get(0));
+            AggregateFunctionCallExpression aggFuncExpr = AsterixBuiltinFunctions.makeAggregateFunctionExpression(
+                    aggFuncIdent, aggArgs);
+            LogicalVariable newVar = context.newVar();
+            aggOp.getVariables().add(newVar);
+            aggOp.getExpressions().add(new MutableObject<ILogicalExpression>(aggFuncExpr));
+
+            // The assign now just "renames" the variable to make sure the upstream plan still works.
+            srcAssignExprRef.setValue(new VariableReferenceExpression(newVar));
+        }
+
+        context.computeAndSetTypeEnvironmentForOperator(aggOp);
+        context.computeAndSetTypeEnvironmentForOperator(assignOp);
+        return true;
+    }
+
+    private boolean fingAggFuncExprRef(List<Mutable<ILogicalExpression>> exprRefs, LogicalVariable aggVar,
+            List<Mutable<ILogicalExpression>> srcAssignExprRefs) {
+        for (Mutable<ILogicalExpression> exprRef : exprRefs) {
+            ILogicalExpression expr = exprRef.getValue();
+
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                if (((VariableReferenceExpression) expr).getVariableReference().equals(aggVar)) {
+                    return false;
+                }
+            }
+
+            if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+                continue;
+            }
+            AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+            FunctionIdentifier funcIdent = AsterixBuiltinFunctions.getAggregateFunction(funcExpr
+                    .getFunctionIdentifier());
+            if (funcIdent == null) {
+                // Recursively look in func args.
+                if (fingAggFuncExprRef(funcExpr.getArguments(), aggVar, srcAssignExprRefs) == false) {
+                    return false;
+                }
+
+            } else {
+                // Check if this is the expr that uses aggVar.
+                Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
+                funcExpr.getUsedVariables(usedVars);
+                if (usedVars.contains(aggVar)) {
+                    srcAssignExprRefs.add(exprRef);
+                }
+            }
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java
new file mode 100644
index 0000000..90b2f25
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoGroupbyRule.java
@@ -0,0 +1,462 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushAggregateIntoGroupbyRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        Map<LogicalVariable, Integer> gbyAggVars = new HashMap<LogicalVariable, Integer>();
+        Map<LogicalVariable, Integer> gbyAggVarToPlanIndex = new HashMap<LogicalVariable, Integer>();
+        Map<LogicalVariable, GroupByOperator> gbyWithAgg = new HashMap<LogicalVariable, GroupByOperator>();
+        Map<ILogicalExpression, ILogicalExpression> aggExprToVarExpr = new HashMap<ILogicalExpression, ILogicalExpression>();
+        // first collect vars. referring to listified sequences
+        boolean changed = collectVarsBottomUp(opRef, context, gbyAggVars, gbyWithAgg, gbyAggVarToPlanIndex,
+                aggExprToVarExpr);
+        if (changed) {
+            removeRedundantListifies(opRef, context, gbyAggVars, gbyWithAgg, gbyAggVarToPlanIndex);
+        }
+        return changed;
+    }
+
+    private void removeRedundantListifies(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
+            Map<LogicalVariable, Integer> gbyAggVars, Map<LogicalVariable, GroupByOperator> gbyWithAgg,
+            Map<LogicalVariable, Integer> gbyAggVarToPlanIndex) throws AlgebricksException {
+        for (LogicalVariable aggVar : gbyAggVars.keySet()) {
+            int occurs = gbyAggVars.get(aggVar);
+            if (occurs == 0) {
+                GroupByOperator gbyOp = gbyWithAgg.get(aggVar);
+                AggregateOperator aggOp = (AggregateOperator) gbyOp.getNestedPlans()
+                        .get(gbyAggVarToPlanIndex.get(aggVar)).getRoots().get(0).getValue();
+                int pos = aggOp.getVariables().indexOf(aggVar);
+                if (pos >= 0) {
+                    aggOp.getVariables().remove(pos);
+                    aggOp.getExpressions().remove(pos);
+                    List<LogicalVariable> producedVarsAtAgg = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getProducedVariablesInDescendantsAndSelf(aggOp, producedVarsAtAgg);
+                    if (producedVarsAtAgg.isEmpty()) {
+                        gbyOp.getNestedPlans().remove(gbyAggVarToPlanIndex.get(aggVar));
+                    }
+                }
+            }
+        }
+    }
+
+    private boolean collectVarsBottomUp(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
+            Map<LogicalVariable, Integer> gbyListifyVarsCount, Map<LogicalVariable, GroupByOperator> gbyWithAgg,
+            Map<LogicalVariable, Integer> gbyAggVarToPlanIndex,
+            Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr) throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        context.addToDontApplySet(this, op1);
+        boolean change = false;
+        for (Mutable<ILogicalOperator> child : op1.getInputs()) {
+            if (collectVarsBottomUp(child, context, gbyListifyVarsCount, gbyWithAgg, gbyAggVarToPlanIndex,
+                    aggregateExprToVarExpr)) {
+                change = true;
+            }
+        }
+        // Need to use a list instead of a hash-set, because a var. may appear
+        // several times in the same op.
+        List<LogicalVariable> used = new LinkedList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op1, used);
+        switch (op1.getOperatorTag()) {
+            case ASSIGN:
+            case SELECT: {
+                boolean found = false;
+                // Do some prefiltering: check if the Assign uses any gby vars.
+                for (LogicalVariable v : used) {
+                    if (gbyListifyVarsCount.get(v) != null) {
+                        found = true;
+                        break;
+                    }
+                }
+                if (found) {
+                    if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+                        AssignOperator assign = (AssignOperator) op1;
+                        for (Mutable<ILogicalExpression> exprRef : assign.getExpressions()) {
+                            Pair<Boolean, ILogicalExpression> p = extractAggFunctionsFromExpression(exprRef,
+                                    gbyWithAgg, aggregateExprToVarExpr, context);
+                            if (p.first) {
+                                change = true;
+                                exprRef.setValue(p.second);
+                            }
+                        }
+                    }
+                    if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
+                        SelectOperator select = (SelectOperator) op1;
+                        Mutable<ILogicalExpression> exprRef = select.getCondition();
+                        Pair<Boolean, ILogicalExpression> p = extractAggFunctionsFromExpression(exprRef, gbyWithAgg,
+                                aggregateExprToVarExpr, context);
+                        if (p.first) {
+                            change = true;
+                            exprRef.setValue(p.second);
+                        }
+                    }
+                    used.clear();
+                    VariableUtilities.getUsedVariables(op1, used);
+                    // increment the count for the ones which are still used
+                    for (LogicalVariable v : used) {
+                        Integer m = gbyListifyVarsCount.get(v);
+                        if (m != null) {
+                            gbyListifyVarsCount.put(v, m + 1);
+                        }
+                    }
+                }
+                break;
+            }
+            case SUBPLAN: {
+                for (LogicalVariable v : used) {
+                    Integer m = gbyListifyVarsCount.get(v);
+                    if (m != null) {
+                        GroupByOperator gbyOp = gbyWithAgg.get(v);
+                        if (pushSubplanAsAggIntoGby(opRef, gbyOp, v, gbyListifyVarsCount, gbyWithAgg,
+                                gbyAggVarToPlanIndex, context)) {
+                            change = true;
+                        } else {
+                            gbyListifyVarsCount.put(v, m + 1);
+                        }
+                        break;
+                    }
+                }
+                break;
+            }
+            case GROUP: {
+                List<LogicalVariable> vars = collectOneVarPerAggFromGroupOp((GroupByOperator) op1);
+                if (vars != null) {
+                    for (int i = 0; i < vars.size(); i++) {
+                        LogicalVariable v = vars.get(i);
+                        if (v != null) {
+                            gbyListifyVarsCount.put(v, 0);
+                            gbyAggVarToPlanIndex.put(v, i);
+                            gbyWithAgg.put(v, (GroupByOperator) op1);
+                        }
+                    }
+                }
+                break;
+            }
+            default: {
+                for (LogicalVariable v : used) {
+                    Integer m = gbyListifyVarsCount.get(v);
+                    if (m != null) {
+                        gbyListifyVarsCount.put(v, m + 1);
+                    }
+                }
+            }
+        }
+        return change;
+    }
+
+    private List<LogicalVariable> collectOneVarPerAggFromGroupOp(GroupByOperator group) {
+        List<ILogicalPlan> nPlans = group.getNestedPlans();
+        if (nPlans == null || nPlans.size() < 1) {
+            return null;
+        }
+
+        List<LogicalVariable> aggVars = new ArrayList<LogicalVariable>();
+        // test that the group-by computes a "listify" aggregate
+        for (int i = 0; i < nPlans.size(); i++) {
+            AbstractLogicalOperator topOp = (AbstractLogicalOperator) nPlans.get(i).getRoots().get(0).getValue();
+            if (topOp.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+                continue;
+            }
+            AggregateOperator agg = (AggregateOperator) topOp;
+            if (agg.getVariables().size() != 1) {
+                continue;
+            }
+            ILogicalExpression expr = agg.getExpressions().get(0).getValue();
+            if (((AbstractLogicalExpression) expr).getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+                continue;
+            }
+            AbstractFunctionCallExpression fceAgg = (AbstractFunctionCallExpression) expr;
+            if (fceAgg.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
+                continue;
+            }
+            aggVars.add(agg.getVariables().get(0));
+        }
+        return aggVars;
+    }
+
+    /**
+     * @param expr
+     * @param aggVars
+     * @param gbyWithAgg
+     * @param context
+     * @return a pair whose first member is a boolean which is true iff
+     *         something was changed in the expression tree rooted at expr. The
+     *         second member is the result of transforming expr.
+     * @throws AlgebricksException
+     */
+    private Pair<Boolean, ILogicalExpression> extractAggFunctionsFromExpression(Mutable<ILogicalExpression> exprRef,
+            Map<LogicalVariable, GroupByOperator> gbyWithAgg,
+            Map<ILogicalExpression, ILogicalExpression> aggregateExprToVarExpr, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalExpression expr = exprRef.getValue();
+        switch (expr.getExpressionTag()) {
+            case FUNCTION_CALL: {
+                AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
+                FunctionIdentifier fi = AsterixBuiltinFunctions.getAggregateFunction(fce.getFunctionIdentifier());
+                if (fi != null) {
+                    ILogicalExpression a1 = fce.getArguments().get(0).getValue();
+                    if (a1.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                        LogicalVariable argVar = ((VariableReferenceExpression) a1).getVariableReference();
+                        GroupByOperator gbyOp = gbyWithAgg.get(argVar);
+
+                        if (gbyOp != null) {
+                            if (!aggregateExprToVarExpr.containsKey(expr)) {
+                                LogicalVariable newVar = context.newVar();
+                                AggregateFunctionCallExpression aggFun = AsterixBuiltinFunctions
+                                        .makeAggregateFunctionExpression(fi, fce.getArguments());
+                                rewriteGroupByAggregate(argVar, gbyOp, aggFun, newVar, context);
+                                ILogicalExpression newVarExpr = new VariableReferenceExpression(newVar);
+                                aggregateExprToVarExpr.put(expr, newVarExpr);
+                                return new Pair<Boolean, ILogicalExpression>(Boolean.TRUE, newVarExpr);
+                            } else {
+                                ILogicalExpression varExpr = aggregateExprToVarExpr.get(expr);
+                                return new Pair<Boolean, ILogicalExpression>(Boolean.TRUE, varExpr);
+                            }
+                        }
+                    }
+                }
+
+                boolean change = false;
+                for (Mutable<ILogicalExpression> a : fce.getArguments()) {
+                    Pair<Boolean, ILogicalExpression> aggArg = extractAggFunctionsFromExpression(a, gbyWithAgg,
+                            aggregateExprToVarExpr, context);
+                    if (aggArg.first.booleanValue()) {
+                        a.setValue(aggArg.second);
+                        change = true;
+                    }
+                }
+                return new Pair<Boolean, ILogicalExpression>(change, fce);
+            }
+            case VARIABLE:
+            case CONSTANT: {
+                return new Pair<Boolean, ILogicalExpression>(Boolean.FALSE, expr);
+            }
+            default: {
+                throw new IllegalArgumentException();
+            }
+        }
+    }
+
+    private void rewriteGroupByAggregate(LogicalVariable oldAggVar, GroupByOperator gbyOp,
+            AggregateFunctionCallExpression aggFun, LogicalVariable newAggVar, IOptimizationContext context)
+            throws AlgebricksException {
+        for (int j = 0; j < gbyOp.getNestedPlans().size(); j++) {
+            AggregateOperator aggOp = (AggregateOperator) gbyOp.getNestedPlans().get(j).getRoots().get(0).getValue();
+            int n = aggOp.getVariables().size();
+            for (int i = 0; i < n; i++) {
+                LogicalVariable v = aggOp.getVariables().get(i);
+                if (v.equals(oldAggVar)) {
+                    AbstractFunctionCallExpression oldAggExpr = (AbstractFunctionCallExpression) aggOp.getExpressions()
+                            .get(i).getValue();
+                    AggregateFunctionCallExpression newAggFun = AsterixBuiltinFunctions
+                            .makeAggregateFunctionExpression(aggFun.getFunctionIdentifier(),
+                                    new ArrayList<Mutable<ILogicalExpression>>());
+                    for (Mutable<ILogicalExpression> arg : oldAggExpr.getArguments()) {
+                        ILogicalExpression cloned = ((AbstractLogicalExpression) arg.getValue()).cloneExpression();
+                        newAggFun.getArguments().add(new MutableObject<ILogicalExpression>(cloned));
+                    }
+                    aggOp.getVariables().add(newAggVar);
+                    aggOp.getExpressions().add(new MutableObject<ILogicalExpression>(newAggFun));
+                    context.computeAndSetTypeEnvironmentForOperator(aggOp);
+                    break;
+                }
+            }
+        }
+    }
+
+    private boolean pushSubplanAsAggIntoGby(Mutable<ILogicalOperator> subplanOpRef, GroupByOperator gbyOp,
+            LogicalVariable varFromGroupAgg, Map<LogicalVariable, Integer> gbyAggVars,
+            Map<LogicalVariable, GroupByOperator> gbyWithAgg, Map<LogicalVariable, Integer> gbyAggVarToPlanIndex,
+            IOptimizationContext context) throws AlgebricksException {
+        SubplanOperator subplan = (SubplanOperator) subplanOpRef.getValue();
+        // only free var can be varFromGroupAgg
+        HashSet<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
+        for (LogicalVariable vFree : freeVars) {
+            if (!vFree.equals(varFromGroupAgg)) {
+                return false;
+            }
+        }
+
+        List<ILogicalPlan> plans = subplan.getNestedPlans();
+        if (plans.size() > 1) {
+            return false;
+        }
+        ILogicalPlan p = plans.get(0);
+        if (p.getRoots().size() > 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef = p.getRoots().get(0);
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AggregateOperator aggInSubplanOp = (AggregateOperator) op;
+        LogicalVariable unnestVar = null;
+        boolean pushableNestedSubplan = false;
+        while (op.getInputs().size() == 1) {
+            opRef = op.getInputs().get(0);
+            op = (AbstractLogicalOperator) opRef.getValue();
+            switch (op.getOperatorTag()) {
+                case ASSIGN: {
+                    break;
+                }
+                case UNNEST: {
+                    UnnestOperator unnest = (UnnestOperator) op;
+                    if (unnest.getPositionalVariable() != null) {
+                        // TODO currently subplan with both accumulating and running aggregate is not supported.
+                        return false;
+                    }
+                    ILogicalExpression expr = unnest.getExpressionRef().getValue();
+                    if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+                        return false;
+                    }
+                    AbstractFunctionCallExpression fun = (AbstractFunctionCallExpression) expr;
+                    if (fun.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
+                        return false;
+                    }
+                    ILogicalExpression arg0 = fun.getArguments().get(0).getValue();
+                    if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                        return false;
+                    }
+                    VariableReferenceExpression varExpr = (VariableReferenceExpression) arg0;
+                    if (!varExpr.getVariableReference().equals(varFromGroupAgg)) {
+                        return false;
+                    }
+                    opRef = op.getInputs().get(0);
+                    op = (AbstractLogicalOperator) opRef.getValue();
+                    if (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                        return false;
+                    }
+                    pushableNestedSubplan = true;
+                    unnestVar = unnest.getVariable();
+                    break;
+                }
+                default: {
+                    return false;
+                }
+            }
+        }
+        if (pushableNestedSubplan) {
+            for (int i = 0; i < gbyOp.getNestedPlans().size(); i++) {
+                Mutable<ILogicalOperator> gbyAggRef = gbyOp.getNestedPlans().get(i).getRoots().get(0);
+                AggregateOperator gbyAgg = (AggregateOperator) gbyAggRef.getValue();
+                Mutable<ILogicalOperator> gbyAggChildRef = gbyAgg.getInputs().get(0);
+                OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar,
+                        findListifiedVariable(gbyAgg, varFromGroupAgg), true, context);
+                gbyAgg.getVariables().addAll(aggInSubplanOp.getVariables());
+                gbyAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
+                for (LogicalVariable v : aggInSubplanOp.getVariables()) {
+                    gbyWithAgg.put(v, gbyOp);
+                    gbyAggVars.put(v, 0);
+                    gbyAggVarToPlanIndex.put(v, i);
+                }
+
+                Mutable<ILogicalOperator> opRef1InSubplan = aggInSubplanOp.getInputs().get(0);
+                if (opRef1InSubplan.getValue().getInputs().size() > 0) {
+                    Mutable<ILogicalOperator> opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
+                    AbstractLogicalOperator op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
+                    if (op2InSubplan.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                        List<Mutable<ILogicalOperator>> gbyInpList = gbyAgg.getInputs();
+                        gbyInpList.clear();
+                        gbyInpList.add(opRef1InSubplan);
+                        while (true) {
+                            opRef2InSubplan = opRef1InSubplan.getValue().getInputs().get(0);
+                            op2InSubplan = (AbstractLogicalOperator) opRef2InSubplan.getValue();
+                            if (op2InSubplan.getOperatorTag() == LogicalOperatorTag.UNNEST) {
+                                List<Mutable<ILogicalOperator>> opInpList = opRef1InSubplan.getValue().getInputs();
+                                opInpList.clear();
+                                opInpList.add(gbyAggChildRef);
+                                break;
+                            }
+                            opRef1InSubplan = opRef2InSubplan;
+                            if (opRef1InSubplan.getValue().getInputs().size() == 0) {
+                                throw new IllegalStateException("PushAggregateIntoGroupbyRule: could not find UNNEST.");
+                            }
+                        }
+                    }
+                }
+                subplanOpRef.setValue(subplan.getInputs().get(0).getValue());
+                OperatorPropertiesUtil.typeOpRec(gbyAggRef, context);
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private LogicalVariable findListifiedVariable(AggregateOperator gbyAgg, LogicalVariable varFromGroupAgg) {
+        int n = gbyAgg.getVariables().size();
+
+        for (int i = 0; i < n; i++) {
+            if (gbyAgg.getVariables().get(i).equals(varFromGroupAgg)) {
+                AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) gbyAgg.getExpressions().get(i)
+                        .getValue();
+                if (fce.getFunctionIdentifier().equals(AsterixBuiltinFunctions.LISTIFY)) {
+                    ILogicalExpression argExpr = fce.getArguments().get(0).getValue();
+                    if (((AbstractLogicalExpression) argExpr).getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                        return ((VariableReferenceExpression) argExpr).getVariableReference();
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
new file mode 100644
index 0000000..3a19854
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
@@ -0,0 +1,391 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.algebra.base.AsterixOperatorAnnotations;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.exceptions.AsterixRuntimeException;
+import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
+import edu.uci.ics.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.declared.AqlSourceId;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.base.IAObject;
+import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.optimizer.base.AnalysisUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushFieldAccessRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (context.checkIfInDontApplySet(this, op)) {
+            return false;
+        }
+        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator access = (AssignOperator) op;
+        ILogicalExpression expr = getFirstExpr(access);
+        String finalAnnot = null;
+        if (AnalysisUtil.isAccessToFieldRecord(expr)) {
+            finalAnnot = AsterixOperatorAnnotations.PUSHED_FIELD_ACCESS;
+        } else if (AnalysisUtil.isRunnableAccessToFieldRecord(expr)) {
+            finalAnnot = AsterixOperatorAnnotations.PUSHED_RUNNABLE_FIELD_ACCESS;
+        } else {
+            return false;
+        }
+        return propagateFieldAccessRec(opRef, context, finalAnnot);
+    }
+
+    @SuppressWarnings("unchecked")
+    private boolean isAccessToIndexedField(AssignOperator assign, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) assign.getExpressions().get(0)
+                .getValue();
+        ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
+        if (e0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            return false;
+        }
+        LogicalVariable var = ((VariableReferenceExpression) e0).getVariableReference();
+        if (context.findPrimaryKey(var) == null) {
+            // not referring to a dataset record
+            return false;
+        }
+        AbstractLogicalOperator op = assign;
+        while (op.getInputs().size() == 1 && op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
+            op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        }
+        if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
+            return false;
+        }
+        DataSourceScanOperator scan = (DataSourceScanOperator) op;
+        LogicalVariable recVar = scan.getVariables().get(scan.getVariables().size() - 1);
+        if (recVar != var) {
+            return false;
+        }
+        AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
+        AqlSourceId asid = ((IDataSource<AqlSourceId>) scan.getDataSource()).getId();
+
+        Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
+        if (dataset == null) {
+            throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
+        }
+        if (dataset.getDatasetType() != DatasetType.INTERNAL) {
+            return false;
+        }
+        ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
+        if (e1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+            return false;
+        }
+        ConstantExpression ce = (ConstantExpression) e1;
+        IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
+        String fldName;
+        if (obj.getType().getTypeTag() == ATypeTag.STRING) {
+            fldName = ((AString) obj).getStringValue();
+        } else {
+            int pos = ((AInt32) obj).getIntegerValue();
+            String tName = dataset.getItemTypeName();
+            IAType t = mp.findType(dataset.getDataverseName(), tName);
+            if (t.getTypeTag() != ATypeTag.RECORD) {
+                return false;
+            }
+            ARecordType rt = (ARecordType) t;
+            if (pos >= rt.getFieldNames().length) {
+                return false;
+            }
+            fldName = rt.getFieldNames()[pos];
+        }
+
+        List<Index> datasetIndexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
+        boolean hasSecondaryIndex = false;
+        for (Index index : datasetIndexes) {
+            if (index.isSecondaryIndex()) {
+                hasSecondaryIndex = true;
+                break;
+            }
+        }
+
+        return hasSecondaryIndex;
+    }
+
+    private boolean tryingToPushThroughSelectionWithSameDataSource(AssignOperator access, AbstractLogicalOperator op2) {
+        if (op2.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+        ILogicalExpression e1 = (ILogicalExpression) access.getAnnotations().get(
+                AsterixOperatorAnnotations.FIELD_ACCESS);
+        if (e1 == null) {
+            return false;
+        }
+        ILogicalExpression e2 = (ILogicalExpression) op2.getAnnotations().get(AsterixOperatorAnnotations.FIELD_ACCESS);
+        if (e2 == null) {
+            return false;
+        }
+        return e1.equals(e2);
+    }
+
+    @SuppressWarnings("unchecked")
+    private boolean propagateFieldAccessRec(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
+            String finalAnnot) throws AlgebricksException {
+        AssignOperator access = (AssignOperator) opRef.getValue();
+        Mutable<ILogicalOperator> opRef2 = access.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        // If it's not an indexed field, it is pushed so that scan can be
+        // rewritten into index search.
+        if (op2.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(access, op2)
+                && !(op2.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(access, context))) {
+            return false;
+        }
+        if (tryingToPushThroughSelectionWithSameDataSource(access, op2)) {
+            return false;
+        }
+        if (testAndModifyRedundantOp(access, op2)) {
+            propagateFieldAccessRec(opRef2, context, finalAnnot);
+            return true;
+        }
+        List<LogicalVariable> usedInAccess = new LinkedList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(access, usedInAccess);
+        List<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+        if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            VariableUtilities.getLiveVariables(op2, produced2);
+        } else {
+            VariableUtilities.getProducedVariables(op2, produced2);
+        }
+        boolean pushItDown = false;
+        List<LogicalVariable> inter = new ArrayList<LogicalVariable>(usedInAccess);
+        if (inter.isEmpty()) { // ground value
+            return false;
+        }
+        inter.retainAll(produced2);
+        if (inter.isEmpty()) {
+            pushItDown = true;
+        } else if (op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            GroupByOperator g = (GroupByOperator) op2;
+            List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<Pair<LogicalVariable, LogicalVariable>>();
+            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
+                ILogicalExpression e = p.second.getValue();
+                if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                    LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
+                    if (inter.contains(decorVar)) {
+                        inter.remove(decorVar);
+                        LogicalVariable v1 = ((VariableReferenceExpression) e).getVariableReference();
+                        varMappings.add(new Pair<LogicalVariable, LogicalVariable>(decorVar, v1));
+                    }
+                }
+            }
+            if (inter.isEmpty()) {
+                boolean changed = false;
+                for (Pair<LogicalVariable, LogicalVariable> m : varMappings) {
+                    LogicalVariable v2 = context.newVar();
+                    LogicalVariable oldVar = access.getVariables().get(0);
+                    g.getDecorList().add(
+                            new Pair<LogicalVariable, Mutable<ILogicalExpression>>(oldVar,
+                                    new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v2))));
+                    changed = true;
+                    access.getVariables().set(0, v2);
+                    VariableUtilities.substituteVariables(access, m.first, m.second, context);
+                }
+                if (changed) {
+                    context.computeAndSetTypeEnvironmentForOperator(g);
+                }
+                usedInAccess.clear();
+                VariableUtilities.getUsedVariables(access, usedInAccess);
+                pushItDown = true;
+            }
+        }
+        if (pushItDown) {
+            if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) op2).getDataSourceReference()
+                        .getValue().getInputs().get(0);
+                pushAccessDown(opRef, op2, childOfSubplan, context, finalAnnot);
+                return true;
+            }
+            if (op2.getInputs().size() == 1 && !op2.hasNestedPlans()) {
+                pushAccessDown(opRef, op2, op2.getInputs().get(0), context, finalAnnot);
+                return true;
+            } else {
+                for (Mutable<ILogicalOperator> inp : op2.getInputs()) {
+                    HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
+                    VariableUtilities.getLiveVariables(inp.getValue(), v2);
+                    if (v2.containsAll(usedInAccess)) {
+                        pushAccessDown(opRef, op2, inp, context, finalAnnot);
+                        return true;
+                    }
+                }
+            }
+            if (op2.hasNestedPlans()) {
+                AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) op2;
+                for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
+                    for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+                        HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
+                        VariableUtilities.getLiveVariables(root.getValue(), v2);
+                        if (v2.containsAll(usedInAccess)) {
+                            pushAccessDown(opRef, op2, root, context, finalAnnot);
+                            return true;
+                        }
+                    }
+                }
+            }
+            throw new AsterixRuntimeException("Field access " + access.getExpressions().get(0).getValue()
+                    + " does not correspond to any input of operator " + op2);
+        } else {
+            // Check if the accessed field is not one of the partitioning key
+            // fields. If yes, we can equate the two variables.
+            if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+                DataSourceScanOperator scan = (DataSourceScanOperator) op2;
+                int n = scan.getVariables().size();
+                LogicalVariable scanRecordVar = scan.getVariables().get(n - 1);
+                AbstractFunctionCallExpression accessFun = (AbstractFunctionCallExpression) access.getExpressions()
+                        .get(0).getValue();
+                ILogicalExpression e0 = accessFun.getArguments().get(0).getValue();
+                LogicalExpressionTag tag = e0.getExpressionTag();
+                if (tag == LogicalExpressionTag.VARIABLE) {
+                    VariableReferenceExpression varRef = (VariableReferenceExpression) e0;
+                    if (varRef.getVariableReference() == scanRecordVar) {
+                        ILogicalExpression e1 = accessFun.getArguments().get(1).getValue();
+                        if (e1.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+                            IDataSource<AqlSourceId> dataSource = (IDataSource<AqlSourceId>) scan.getDataSource();
+                            AqlDataSourceType dsType = ((AqlDataSource) dataSource).getDatasourceType();
+                            if (dsType == AqlDataSourceType.FEED || dsType == AqlDataSourceType.LOADABLE) {
+                                return false;
+                            }
+                            AqlSourceId asid = dataSource.getId();
+                            AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
+                            Dataset dataset = mp.findDataset(asid.getDataverseName(), asid.getDatasourceName());
+                            if (dataset == null) {
+                                throw new AlgebricksException("Dataset " + asid.getDatasourceName() + " not found.");
+                            }
+                            if (dataset.getDatasetType() != DatasetType.INTERNAL) {
+                                setAsFinal(access, context, finalAnnot);
+                                return false;
+                            }
+                            ConstantExpression ce = (ConstantExpression) e1;
+                            IAObject obj = ((AsterixConstantValue) ce.getValue()).getObject();
+                            String fldName;
+                            if (obj.getType().getTypeTag() == ATypeTag.STRING) {
+                                fldName = ((AString) obj).getStringValue();
+                            } else {
+                                int pos = ((AInt32) obj).getIntegerValue();
+                                String tName = dataset.getItemTypeName();
+                                IAType t = mp.findType(dataset.getDataverseName(), tName);
+                                if (t.getTypeTag() != ATypeTag.RECORD) {
+                                    return false;
+                                }
+                                ARecordType rt = (ARecordType) t;
+                                if (pos >= rt.getFieldNames().length) {
+                                    setAsFinal(access, context, finalAnnot);
+                                    return false;
+                                }
+                                fldName = rt.getFieldNames()[pos];
+                            }
+                            int p = DatasetUtils.getPositionOfPartitioningKeyField(dataset, fldName);
+                            if (p < 0) { // not one of the partitioning fields
+                                setAsFinal(access, context, finalAnnot);
+                                return false;
+                            }
+                            LogicalVariable keyVar = scan.getVariables().get(p);
+                            access.getExpressions().get(0).setValue(new VariableReferenceExpression(keyVar));
+                            return true;
+                        }
+                    }
+                }
+            }
+            setAsFinal(access, context, finalAnnot);
+            return false;
+        }
+    }
+
+    private void setAsFinal(ILogicalOperator access, IOptimizationContext context, String finalAnnot) {
+        access.getAnnotations().put(finalAnnot, true);
+        context.addToDontApplySet(this, access);
+    }
+
+    private boolean testAndModifyRedundantOp(AssignOperator access, AbstractLogicalOperator op2) {
+        if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator a2 = (AssignOperator) op2;
+        if (getFirstExpr(access).equals(getFirstExpr(a2))) {
+            access.getExpressions().get(0).setValue(new VariableReferenceExpression(a2.getVariables().get(0)));
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    // indirect recursivity with propagateFieldAccessRec
+    private void pushAccessDown(Mutable<ILogicalOperator> fldAccessOpRef, ILogicalOperator op2,
+            Mutable<ILogicalOperator> inputOfOp2, IOptimizationContext context, String finalAnnot)
+            throws AlgebricksException {
+        ILogicalOperator fieldAccessOp = fldAccessOpRef.getValue();
+        fldAccessOpRef.setValue(op2);
+        List<Mutable<ILogicalOperator>> faInpList = fieldAccessOp.getInputs();
+        faInpList.clear();
+        faInpList.add(new MutableObject<ILogicalOperator>(inputOfOp2.getValue()));
+        inputOfOp2.setValue(fieldAccessOp);
+        // typing
+        context.computeAndSetTypeEnvironmentForOperator(fieldAccessOp);
+        context.computeAndSetTypeEnvironmentForOperator(op2);
+        propagateFieldAccessRec(inputOfOp2, context, finalAnnot);
+    }
+
+    private ILogicalExpression getFirstExpr(AssignOperator assign) {
+        return assign.getExpressions().get(0).getValue();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
new file mode 100644
index 0000000..647c3ee
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
@@ -0,0 +1,172 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushGroupByThroughProduct implements IAlgebraicRewriteRule {
+
+    private enum PushTestResult {
+        FALSE,
+        TRUE,
+        REPEATED_DECORS
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+        InnerJoinOperator join = (InnerJoinOperator) op2;
+        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
+            // not a product
+            return false;
+        }
+        GroupByOperator gby = (GroupByOperator) op1;
+
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorToPush = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorNotToPush = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+
+        Mutable<ILogicalOperator> opLeftRef = join.getInputs().get(0);
+        ILogicalOperator opLeft = opLeftRef.getValue();
+        switch (canPushThrough(gby, opLeft, decorToPush, decorNotToPush)) {
+            case REPEATED_DECORS: {
+                return false;
+            }
+            case TRUE: {
+                push(opRef, opRef2, 0, decorToPush, decorNotToPush, context);
+                return true;
+            }
+            case FALSE: {
+                decorToPush.clear();
+                Mutable<ILogicalOperator> opRightRef = join.getInputs().get(1);
+                ILogicalOperator opRight = opRightRef.getValue();
+                if (canPushThrough(gby, opRight, decorToPush, decorNotToPush) == PushTestResult.TRUE) {
+                    push(opRef, opRef2, 1, decorToPush, decorNotToPush, context);
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+            default: {
+                throw new IllegalStateException();
+            }
+        }
+    }
+
+    private void push(Mutable<ILogicalOperator> opRefGby, Mutable<ILogicalOperator> opRefJoin, int branch,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorToPush,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorNotToPush, IOptimizationContext context)
+            throws AlgebricksException {
+        GroupByOperator gby = (GroupByOperator) opRefGby.getValue();
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) opRefJoin.getValue();
+        gby.getDecorList().clear();
+        gby.getDecorList().addAll(decorToPush);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorNotToPush) {
+            LogicalVariable v1 = p.first;
+            VariableReferenceExpression varRef = (VariableReferenceExpression) p.second.getValue();
+            LogicalVariable v2 = varRef.getVariableReference();
+            OperatorManipulationUtil.substituteVarRec(join, v2, v1, true, context);
+        }
+        Mutable<ILogicalOperator> branchRef = join.getInputs().get(branch);
+        ILogicalOperator opBranch = branchRef.getValue();
+        opRefJoin.setValue(opBranch);
+        branchRef.setValue(gby);
+        opRefGby.setValue(join);
+    }
+
+    private PushTestResult canPushThrough(GroupByOperator gby, ILogicalOperator branch,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> toPush,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> notToPush) throws AlgebricksException {
+        Collection<LogicalVariable> fromBranch = new HashSet<LogicalVariable>();
+        VariableUtilities.getLiveVariables(branch, fromBranch);
+        Collection<LogicalVariable> usedInGbyExprList = new ArrayList<LogicalVariable>();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList()) {
+            p.second.getValue().getUsedVariables(usedInGbyExprList);
+        }
+
+        if (!fromBranch.containsAll(usedInGbyExprList)) {
+            return PushTestResult.FALSE;
+        }
+        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+        for (ILogicalPlan p : gby.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), free);
+            }
+        }
+        if (!fromBranch.containsAll(free)) {
+            return PushTestResult.FALSE;
+        }
+
+        Set<LogicalVariable> decorVarRhs = new HashSet<LogicalVariable>();
+        decorVarRhs.clear();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                return PushTestResult.FALSE;
+            }
+            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+            LogicalVariable v = varRef.getVariableReference();
+            if (decorVarRhs.contains(v)) {
+                return PushTestResult.REPEATED_DECORS;
+            }
+            decorVarRhs.add(v);
+
+            if (fromBranch.contains(v)) {
+                toPush.add(p);
+            } else {
+                notToPush.add(p);
+            }
+        }
+        return PushTestResult.TRUE;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushProperJoinThroughProduct.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushProperJoinThroughProduct.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushProperJoinThroughProduct.java
new file mode 100644
index 0000000..7d9d261
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushProperJoinThroughProduct.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushProperJoinThroughProduct implements IAlgebraicRewriteRule {
+
+    private List<LogicalVariable> usedInCond1AndMaps = new ArrayList<LogicalVariable>();
+    private List<LogicalVariable> productLeftBranchVars = new ArrayList<LogicalVariable>();
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        LogicalOperatorTag tag1 = op.getOperatorTag();
+        if (tag1 != LogicalOperatorTag.INNERJOIN && tag1 != LogicalOperatorTag.LEFTOUTERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join1 = (AbstractBinaryJoinOperator) op;
+        ILogicalExpression cond1 = join1.getCondition().getValue();
+        // don't try to push a product down
+        if (OperatorPropertiesUtil.isAlwaysTrueCond(cond1)) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        while (op2.isMap()) {
+            opRef2 = op2.getInputs().get(0);
+            op2 = (AbstractLogicalOperator) opRef2.getValue();
+        }
+
+        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+
+        InnerJoinOperator product = (InnerJoinOperator) op2;
+        if (!OperatorPropertiesUtil.isAlwaysTrueCond(product.getCondition().getValue())) {
+            return false;
+        }
+
+        usedInCond1AndMaps.clear();
+        cond1.getUsedVariables(usedInCond1AndMaps);
+        Mutable<ILogicalOperator> opIterRef = op.getInputs().get(0);
+        ILogicalOperator opIter = opIterRef.getValue();
+        do {
+            VariableUtilities.getUsedVariables(opIter, usedInCond1AndMaps);
+            opIterRef = opIter.getInputs().get(0);
+            opIter = opIterRef.getValue();
+        } while (opIter.isMap());
+
+        productLeftBranchVars.clear();
+        ILogicalOperator opLeft = op2.getInputs().get(0).getValue();
+        VariableUtilities.getLiveVariables(opLeft, productLeftBranchVars);
+
+        if (!OperatorPropertiesUtil.disjoint(usedInCond1AndMaps, productLeftBranchVars)) {
+            return false;
+        }
+
+        // now push the operators from in between joins, too
+        opIterRef = op.getInputs().get(0);
+        opIter = opIterRef.getValue();
+
+        Mutable<ILogicalOperator> op3Ref = product.getInputs().get(1);
+        ILogicalOperator op3 = op3Ref.getValue();
+
+        opRef2.setValue(op3);
+        op3Ref.setValue(join1);
+        opRef.setValue(product);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSimilarityFunctionsBelowJoin.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSimilarityFunctionsBelowJoin.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSimilarityFunctionsBelowJoin.java
new file mode 100644
index 0000000..d9e06f4
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushSimilarityFunctionsBelowJoin.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushFunctionsBelowJoin;
+
+/**
+ * Pushes similarity function-call expressions below a join if possible.
+ * Assigns the similarity function-call expressions to new variables, and replaces the original
+ * expression with a corresponding variable reference expression.
+ * This rule can help reduce the cost of computing expensive similarity functions by pushing them below
+ * a join (which may blow up the cardinality).
+ * Also, this rule may help to enable other rules such as common subexpression elimination, again to reduce
+ * the number of calls to expensive similarity functions.
+ * Example:
+ * Before plan:
+ * assign [$$10] <- [funcA(funcB(simFuncX($$3, $$4)))]
+ * join (some condition)
+ * join_branch_0 where $$3 and $$4 are not live
+ * ...
+ * join_branch_1 where $$3 and $$4 are live
+ * ...
+ * After plan:
+ * assign [$$10] <- [funcA(funcB($$11))]
+ * join (some condition)
+ * join_branch_0 where $$3 and $$4 are not live
+ * ...
+ * join_branch_1 where $$3 and $$4 are live
+ * assign[$$11] <- [simFuncX($$3, $$4)]
+ * ...
+ */
+public class PushSimilarityFunctionsBelowJoin extends PushFunctionsBelowJoin {
+
+    private static final Set<FunctionIdentifier> simFuncIdents = new HashSet<FunctionIdentifier>();
+    static {
+        simFuncIdents.add(AsterixBuiltinFunctions.SIMILARITY_JACCARD);
+        simFuncIdents.add(AsterixBuiltinFunctions.SIMILARITY_JACCARD_CHECK);
+        simFuncIdents.add(AsterixBuiltinFunctions.EDIT_DISTANCE);
+        simFuncIdents.add(AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK);
+    }
+
+    public PushSimilarityFunctionsBelowJoin() {
+        super(simFuncIdents);
+    }
+}