[01/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

Repository: incubator-asterixdb-hyracks
Updated Branches:
  refs/heads/master 4db4f57e9 -> 9939b48ec


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java
deleted file mode 100644
index 4ed33f4..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamSelectRuntimeFactory.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.io.DataOutput;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFieldFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-
-public class StreamSelectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final IScalarEvaluatorFactory cond;
-
-    private final IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
-
-    private final boolean retainNull;
-
-    private final int nullPlaceholderVariableIndex;
-
-    private final INullWriterFactory nullWriterFactory;
-
-    /**
-     * @param cond
-     * @param projectionList               if projectionList is null, then no projection is performed
-     * @param retainNull
-     * @param nullPlaceholderVariableIndex
-     * @param nullWriterFactory
-     * @throws HyracksDataException
-     */
-    public StreamSelectRuntimeFactory(IScalarEvaluatorFactory cond, int[] projectionList,
-            IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory, boolean retainNull,
-            int nullPlaceholderVariableIndex, INullWriterFactory nullWriterFactory) {
-        super(projectionList);
-        this.cond = cond;
-        this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
-        this.retainNull = retainNull;
-        this.nullPlaceholderVariableIndex = nullPlaceholderVariableIndex;
-        this.nullWriterFactory = nullWriterFactory;
-    }
-
-    @Override
-    public String toString() {
-        return "stream-select " + cond.toString();
-    }
-
-    @Override
-    public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) {
-        final IBinaryBooleanInspector bbi = binaryBooleanInspectorFactory.createBinaryBooleanInspector(ctx);
-        return new AbstractOneInputOneOutputOneFieldFramePushRuntime() {
-            private IPointable p = VoidPointable.FACTORY.createPointable();
-            private IScalarEvaluator eval;
-            private INullWriter nullWriter = null;
-            private ArrayTupleBuilder nullTupleBuilder = null;
-
-            @Override
-            public void open() throws HyracksDataException {
-                if (eval == null) {
-                    initAccessAppendFieldRef(ctx);
-                    try {
-                        eval = cond.createScalarEvaluator(ctx);
-                    } catch (AlgebricksException ae) {
-                        throw new HyracksDataException(ae);
-                    }
-                }
-                writer.open();
-
-                //prepare nullTupleBuilder
-                if (retainNull && nullWriter == null) {
-                    nullWriter = nullWriterFactory.createNullWriter();
-                    nullTupleBuilder = new ArrayTupleBuilder(1);
-                    DataOutput out = nullTupleBuilder.getDataOutput();
-                    nullWriter.writeNull(out);
-                    nullTupleBuilder.addFieldEndOffset();
-                }
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-                for (int t = 0; t < nTuple; t++) {
-                    tRef.reset(tAccess, t);
-                    try {
-                        eval.evaluate(tRef, p);
-                    } catch (AlgebricksException ae) {
-                        throw new HyracksDataException(ae);
-                    }
-                    if (bbi.getBooleanValue(p.getByteArray(), p.getStartOffset(), p.getLength())) {
-                        if (projectionList != null) {
-                            appendProjectionToFrame(t, projectionList);
-                        } else {
-                            appendTupleToFrame(t);
-                        }
-                    } else {
-                        if (retainNull) {
-                            for (int i = 0; i < tRef.getFieldCount(); i++) {
-                                if (i == nullPlaceholderVariableIndex) {
-                                    appendField(nullTupleBuilder.getByteArray(), 0, nullTupleBuilder.getSize());
-                                } else {
-                                    appendField(tAccess, t, i);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        };
-    }
-
-}

[12/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java
deleted file mode 100644
index 56b2a8e..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractStableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * @author yingyib
- *         merge externalsort+preclustered-gby into sort-gby
- */
-public class PushGroupByIntoSortRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        ILogicalOperator op1 = opRef.getValue();
-        if (op1 == null) {
-            return false;
-        }
-        boolean changed = false;
-        for (Mutable<ILogicalOperator> childRef : op1.getInputs()) {
-            AbstractLogicalOperator op = (AbstractLogicalOperator) childRef.getValue();
-            if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
-                PhysicalOperatorTag opTag = op.getPhysicalOperator().getOperatorTag();
-                GroupByOperator groupByOperator = (GroupByOperator) op;
-                if (opTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
-                    Mutable<ILogicalOperator> op2Ref = op.getInputs().get(0).getValue().getInputs().get(0);
-                    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-                    if (op2.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT) {
-                        AbstractStableSortPOperator sortPhysicalOperator = (AbstractStableSortPOperator) op2
-                                .getPhysicalOperator();
-                        if (groupByOperator.getNestedPlans().size() != 1) {
-                            //Sort group-by currently works only for one nested plan with one root containing
-                            //an aggregate and a nested-tuple-source.
-                            continue;
-                        }
-                        ILogicalPlan p0 = groupByOperator.getNestedPlans().get(0);
-                        if (p0.getRoots().size() != 1) {
-                            //Sort group-by currently works only for one nested plan with one root containing
-                            //an aggregate and a nested-tuple-source.
-                            continue;
-                        }
-
-                        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
-                        AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
-                        if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-                            //we only rewrite aggregation function; do nothing for running aggregates
-                            continue;
-                        }
-                        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
-                        AbstractLogicalOperator aggInputOp = (AbstractLogicalOperator) aggOp.getInputs().get(0)
-                                .getValue();
-                        if (aggInputOp.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
-                            continue;
-                        }
-
-                        boolean hasIntermediateAggregate = generateMergeAggregationExpressions(groupByOperator, context);
-                        if (!hasIntermediateAggregate) {
-                            continue;
-                        }
-
-                        //replace preclustered gby with sort gby
-                        op.setPhysicalOperator(new SortGroupByPOperator(groupByOperator.getGroupByList(), context
-                                .getPhysicalOptimizationConfig().getMaxFramesExternalGroupBy(), sortPhysicalOperator
-                                .getSortColumns()));
-
-                        // remove the stable sort operator
-                        op.getInputs().clear();
-                        op.getInputs().addAll(op2.getInputs());
-                        changed = true;
-                    }
-                }
-                continue;
-            } else {
-                continue;
-            }
-        }
-        return changed;
-    }
-
-    private boolean generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context)
-            throws AlgebricksException {
-        if (gby.getNestedPlans().size() != 1) {
-            throw new AlgebricksException(
-                    "External/sort group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        ILogicalPlan p0 = gby.getNestedPlans().get(0);
-        if (p0.getRoots().size() != 1) {
-            throw new AlgebricksException(
-                    "External/sort group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
-                .getMergeAggregationExpressionFactory();
-        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
-        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
-        List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
-        List<LogicalVariable> originalAggVars = aggOp.getVariables();
-        int n = aggOp.getExpressions().size();
-        List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
-        for (int i = 0; i < n; i++) {
-            ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(
-                    originalAggVars.get(i), aggFuncRefs.get(i).getValue(), context);
-            if (mergeExpr == null) {
-                return false;
-            }
-            mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
-        }
-        aggOp.setMergeExpressions(mergeExpressionRefs);
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java
deleted file mode 100644
index 16a71a4..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushMapOperatorDownThroughProductRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (!op1.isMap()) {
-            return false;
-        }
-        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
-        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
-            return false;
-        }
-
-        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op1, used);
-
-        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
-        ILogicalOperator b0 = b0Ref.getValue();
-        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(b0, b0Scm);
-        if (b0Scm.containsAll(used)) {
-            // push operator on left branch
-            op2Ref.setValue(b0);
-            b0Ref.setValue(op1);
-            opRef.setValue(op2);
-            return true;
-        } else {
-            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
-            ILogicalOperator b1 = b1Ref.getValue();
-            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(b1, b1Scm);
-            if (b1Scm.containsAll(used)) {
-                // push operator on right branch
-                op2Ref.setValue(b1);
-                b1Ref.setValue(op1);
-                opRef.setValue(op2);
-                return true;
-            } else {
-                return false;
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
deleted file mode 100644
index 476096d..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushNestedOrderByUnderPreSortedGroupByRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        if (op.getPhysicalOperator() == null) {
-            return false;
-        }
-        AbstractPhysicalOperator pOp = (AbstractPhysicalOperator) op.getPhysicalOperator();
-        if (pOp.getOperatorTag() != PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
-            return false;
-        }
-        GroupByOperator gby = (GroupByOperator) op;
-        ILogicalPlan plan = gby.getNestedPlans().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) plan.getRoots().get(0).getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-        Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.ORDER) {
-            return false;
-        }
-        OrderOperator order1 = (OrderOperator) op2;
-        if (!isIndependentFromChildren(order1)) {
-            return false;
-        }
-        AbstractPhysicalOperator pOrder1 = (AbstractPhysicalOperator) op2.getPhysicalOperator();
-        if (pOrder1.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT
-                && pOrder1.getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT) {
-            return false;
-        }
-        // StableSortPOperator sort1 = (StableSortPOperator) pOrder1;
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        if (op3.getOperatorTag() != LogicalOperatorTag.ORDER) {
-            return false;
-        }
-        AbstractPhysicalOperator pOp3 = (AbstractPhysicalOperator) op3.getPhysicalOperator();
-        if (pOp3.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) {
-            return false;
-        }
-        OrderOperator order2 = (OrderOperator) op3;
-        StableSortPOperator sort2 = (StableSortPOperator) pOp3;
-        // int n1 = sort1.getSortColumns().length;
-        // int n2 = sort2.getSortColumns().length;
-        // OrderColumn[] sortColumns = new OrderColumn[n2 + n1];
-        // System.arraycopy(sort2.getSortColumns(), 0, sortColumns, 0, n2);
-        // int k = 0;
-        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : order1.getOrderExpressions()) {
-            order2.getOrderExpressions().add(oe);
-            // sortColumns[n2 + k] = sort1.getSortColumns()[k];
-            // ++k;
-        }
-        // sort2.setSortColumns(sortColumns);
-        sort2.computeDeliveredProperties(order2, null);
-        // remove order1
-        ILogicalOperator underOrder1 = order1.getInputs().get(0).getValue();
-        opRef2.setValue(underOrder1);
-        return true;
-    }
-
-    private boolean isIndependentFromChildren(OrderOperator order1) throws AlgebricksException {
-        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
-        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(order1, free);
-        Set<LogicalVariable> usedInOrder = new HashSet<LogicalVariable>();
-        VariableUtilities.getUsedVariables(order1, usedInOrder);
-        return free.containsAll(usedInOrder);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
deleted file mode 100644
index b8ff247..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Pushes projections through its input operator, provided that operator does
- * not produce the projected variables.
- * 
- * @author Nicola
- */
-public class PushProjectDownRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.PROJECT) {
-            return false;
-        }
-        ProjectOperator pi = (ProjectOperator) op;
-        Mutable<ILogicalOperator> opRef2 = pi.getInputs().get(0);
-
-        HashSet<LogicalVariable> toPush = new HashSet<LogicalVariable>();
-        toPush.addAll(pi.getVariables());
-
-        Pair<Boolean, Boolean> p = pushThroughOp(toPush, opRef2, op, context);
-        boolean smthWasPushed = p.first;
-        if (p.second) { // the original projection is redundant
-            opRef.setValue(op.getInputs().get(0).getValue());
-            smthWasPushed = true;
-        }
-
-        return smthWasPushed;
-    }
-
-    private static Pair<Boolean, Boolean> pushThroughOp(HashSet<LogicalVariable> toPush,
-            Mutable<ILogicalOperator> opRef2, ILogicalOperator initialOp, IOptimizationContext context)
-            throws AlgebricksException {
-        List<LogicalVariable> initProjectList = new ArrayList<LogicalVariable>(toPush);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-        do {
-            if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
-                    || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE
-                    || op2.getOperatorTag() == LogicalOperatorTag.PROJECT
-                    || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE
-                    || op2.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
-                return new Pair<Boolean, Boolean>(false, false);
-            }
-            if (!op2.isMap()) {
-                break;
-            }
-            LinkedList<LogicalVariable> usedVars = new LinkedList<LogicalVariable>();
-            VariableUtilities.getUsedVariables(op2, usedVars);
-            toPush.addAll(usedVars);
-            LinkedList<LogicalVariable> producedVars = new LinkedList<LogicalVariable>();
-            VariableUtilities.getProducedVariables(op2, producedVars);
-            toPush.removeAll(producedVars);
-            // we assume pipelineable ops. have only one input
-            opRef2 = op2.getInputs().get(0);
-            op2 = (AbstractLogicalOperator) opRef2.getValue();
-        } while (true);
-
-        LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(op2, produced2);
-        LinkedList<LogicalVariable> used2 = new LinkedList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op2, used2);
-
-        boolean canCommuteProjection = initProjectList.containsAll(toPush) && initProjectList.containsAll(produced2)
-                && initProjectList.containsAll(used2);
-        // if true, we can get rid of the initial projection
-
-        // get rid of useless decor vars.
-        if (!canCommuteProjection && op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
-            boolean gbyChanged = false;
-            GroupByOperator gby = (GroupByOperator) op2;
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
-            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
-                LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
-                if (!toPush.contains(decorVar)) {
-                    used2.remove(decorVar);
-                    gbyChanged = true;
-                } else {
-                    newDecorList.add(p);
-                }
-            }
-            gby.getDecorList().clear();
-            gby.getDecorList().addAll(newDecorList);
-            if (gbyChanged) {
-                context.computeAndSetTypeEnvironmentForOperator(gby);
-            }
-        }
-        used2.clear();
-        VariableUtilities.getUsedVariables(op2, used2);
-
-        toPush.addAll(used2); // remember that toPush is a Set
-        toPush.removeAll(produced2);
-
-        if (toPush.isEmpty()) {
-            return new Pair<Boolean, Boolean>(false, false);
-        }
-
-        boolean smthWasPushed = false;
-        for (Mutable<ILogicalOperator> c : op2.getInputs()) {
-            if (pushNeededProjections(toPush, c, context, initialOp)) {
-                smthWasPushed = true;
-            }
-        }
-        if (op2.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op2;
-            for (ILogicalPlan p : n.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    if (pushNeededProjections(toPush, r, context, initialOp)) {
-                        smthWasPushed = true;
-                    }
-                }
-            }
-        }
-        return new Pair<Boolean, Boolean>(smthWasPushed, canCommuteProjection);
-    }
-
-    // It does not try to push above another Projection.
-    private static boolean pushNeededProjections(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef,
-            IOptimizationContext context, ILogicalOperator initialOp) throws AlgebricksException {
-        HashSet<LogicalVariable> allP = new HashSet<LogicalVariable>();
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        VariableUtilities.getLiveVariables(op, allP);
-
-        HashSet<LogicalVariable> toProject = new HashSet<LogicalVariable>();
-        for (LogicalVariable v : toPush) {
-            if (allP.contains(v)) {
-                toProject.add(v);
-            }
-        }
-        if (toProject.equals(allP)) {
-            // projection would be redundant, since we would project everything
-            // but we can try with the children
-            boolean push = false;
-            if (pushThroughOp(toProject, opRef, initialOp, context).first) {
-                push = true;
-            }
-            return push;
-        } else {
-            return pushAllProjectionsOnTopOf(toProject, opRef, context, initialOp);
-        }
-    }
-
-    // It does not try to push above another Projection.
-    private static boolean pushAllProjectionsOnTopOf(Collection<LogicalVariable> toPush,
-            Mutable<ILogicalOperator> opRef, IOptimizationContext context, ILogicalOperator initialOp)
-            throws AlgebricksException {
-        if (toPush.isEmpty()) {
-            return false;
-        }
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-
-        if (context.checkAndAddToAlreadyCompared(initialOp, op)) {
-            return false;
-        }
-
-        switch (op.getOperatorTag()) {
-            case EXCHANGE: {
-                opRef = opRef.getValue().getInputs().get(0);
-                op = (AbstractLogicalOperator) opRef.getValue();
-                break;
-            }
-            case PROJECT: {
-                return false;
-            }
-        }
-
-        ProjectOperator pi2 = new ProjectOperator(new ArrayList<LogicalVariable>(toPush));
-        pi2.getInputs().add(new MutableObject<ILogicalOperator>(op));
-        opRef.setValue(pi2);
-        pi2.setExecutionMode(op.getExecutionMode());
-        context.computeAndSetTypeEnvironmentForOperator(pi2);
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
deleted file mode 100644
index b17d50f..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushProjectIntoDataSourceScanRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getInputs().size() <= 0)
-            return false;
-        AbstractLogicalOperator project = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        if (project.getOperatorTag() != LogicalOperatorTag.PROJECT)
-            return false;
-        AbstractLogicalOperator exchange = (AbstractLogicalOperator) project.getInputs().get(0).getValue();
-        if (exchange.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
-            return false;
-        AbstractLogicalOperator inputOp = (AbstractLogicalOperator) exchange.getInputs().get(0).getValue();
-        if (inputOp.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
-            return false;
-        DataSourceScanOperator scanOp = (DataSourceScanOperator) inputOp;
-        ProjectOperator projectOp = (ProjectOperator) project;
-        scanOp.addProjectVariables(projectOp.getVariables());
-        if (op.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
-            op.getInputs().set(0, project.getInputs().get(0));
-        } else {
-            op.getInputs().set(0, exchange.getInputs().get(0));
-        }
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
deleted file mode 100644
index cc65996..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushSelectDownRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
-            return false;
-        }
-
-        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-
-        if (context.checkAndAddToAlreadyCompared(op, op2)) {
-            return false;
-        }
-
-        LogicalOperatorTag tag2 = op2.getOperatorTag();
-
-        if (tag2 == LogicalOperatorTag.INNERJOIN || tag2 == LogicalOperatorTag.LEFTOUTERJOIN
-                || tag2 == LogicalOperatorTag.REPLICATE) {
-            return false;
-        } else { // not a join
-            boolean res = propagateSelectionRec(opRef, opRef2);
-            if (res) {
-                OperatorPropertiesUtil.typeOpRec(opRef, context);
-            }
-            return res;
-        }
-    }
-
-    private static boolean propagateSelectionRec(Mutable<ILogicalOperator> sigmaRef, Mutable<ILogicalOperator> opRef2)
-            throws AlgebricksException {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-        if (op2.getInputs().size() != 1 || op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
-            return false;
-        }
-
-        SelectOperator sigma = (SelectOperator) sigmaRef.getValue();
-        LinkedList<LogicalVariable> usedInSigma = new LinkedList<LogicalVariable>();
-        sigma.getCondition().getValue().getUsedVariables(usedInSigma);
-
-        LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(op2, produced2);
-        if (OperatorPropertiesUtil.disjoint(produced2, usedInSigma)) {
-            // just swap
-            opRef2.setValue(sigma);
-            sigmaRef.setValue(op2);
-            List<Mutable<ILogicalOperator>> sigmaInpList = sigma.getInputs();
-            sigmaInpList.clear();
-            sigmaInpList.addAll(op2.getInputs());
-            List<Mutable<ILogicalOperator>> op2InpList = op2.getInputs();
-            op2InpList.clear();
-            op2InpList.add(opRef2);
-            propagateSelectionRec(opRef2, sigma.getInputs().get(0));
-            return true;
-
-        }
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
deleted file mode 100644
index e29299c..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.ListIterator;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushSelectIntoJoinRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        Collection<LogicalVariable> joinLiveVarsLeft = new HashSet<LogicalVariable>();
-        Collection<LogicalVariable> joinLiveVarsRight = new HashSet<LogicalVariable>();
-        Collection<LogicalVariable> liveInOpsToPushLeft = new HashSet<LogicalVariable>();
-        Collection<LogicalVariable> liveInOpsToPushRight = new HashSet<LogicalVariable>();
-
-        List<ILogicalOperator> pushedOnLeft = new ArrayList<ILogicalOperator>();
-        List<ILogicalOperator> pushedOnRight = new ArrayList<ILogicalOperator>();
-        LinkedList<ILogicalOperator> notPushedStack = new LinkedList<ILogicalOperator>();
-        Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
-        Collection<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
-
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
-            return false;
-        }
-        SelectOperator select = (SelectOperator) op;
-        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getValue();
-        AbstractLogicalOperator op2 = son;
-        boolean needToPushOps = false;
-        while (son.isMap()) {
-            needToPushOps = true;
-            Mutable<ILogicalOperator> opRefLink = son.getInputs().get(0);
-            son = (AbstractLogicalOperator) opRefLink.getValue();
-        }
-
-        if (son.getOperatorTag() != LogicalOperatorTag.INNERJOIN
-                && son.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
-            return false;
-        }
-        boolean isLoj = son.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN;
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) son;
-
-        Mutable<ILogicalOperator> joinBranchLeftRef = join.getInputs().get(0);
-        Mutable<ILogicalOperator> joinBranchRightRef = join.getInputs().get(1);
-
-        if (needToPushOps) {
-            ILogicalOperator joinBranchLeft = joinBranchLeftRef.getValue();
-            ILogicalOperator joinBranchRight = joinBranchRightRef.getValue();
-            VariableUtilities.getLiveVariables(joinBranchLeft, joinLiveVarsLeft);
-            VariableUtilities.getLiveVariables(joinBranchRight, joinLiveVarsRight);
-            Mutable<ILogicalOperator> opIterRef = opRef2;
-            ILogicalOperator opIter = op2;
-            while (opIter != join) {
-                LogicalOperatorTag tag = ((AbstractLogicalOperator) opIter).getOperatorTag();
-                if (tag == LogicalOperatorTag.PROJECT) {
-                    notPushedStack.addFirst(opIter);
-                } else {
-                    VariableUtilities.getUsedVariables(opIter, usedVars);
-                    VariableUtilities.getProducedVariables(opIter, producedVars);
-                    if (joinLiveVarsLeft.containsAll(usedVars)) {
-                        pushedOnLeft.add(opIter);
-                        liveInOpsToPushLeft.addAll(producedVars);
-                    } else if (joinLiveVarsRight.containsAll(usedVars)) {
-                        pushedOnRight.add(opIter);
-                        liveInOpsToPushRight.addAll(producedVars);
-                    } else {
-                        return false;
-                    }
-                }
-                opIterRef = opIter.getInputs().get(0);
-                opIter = opIterRef.getValue();
-            }
-            if (isLoj && pushedOnLeft.isEmpty()) {
-                return false;
-            }
-        }
-
-        boolean intersectsAllBranches = true;
-        boolean[] intersectsBranch = new boolean[join.getInputs().size()];
-        LinkedList<LogicalVariable> selectVars = new LinkedList<LogicalVariable>();
-        select.getCondition().getValue().getUsedVariables(selectVars);
-        int i = 0;
-        for (Mutable<ILogicalOperator> branch : join.getInputs()) {
-            LinkedList<LogicalVariable> branchVars = new LinkedList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(branch.getValue(), branchVars);
-            if (i == 0) {
-                branchVars.addAll(liveInOpsToPushLeft);
-            } else {
-                branchVars.addAll(liveInOpsToPushRight);
-            }
-            if (OperatorPropertiesUtil.disjoint(selectVars, branchVars)) {
-                intersectsAllBranches = false;
-            } else {
-                intersectsBranch[i] = true;
-            }
-            i++;
-        }
-        if (!intersectsBranch[0] && !intersectsBranch[1]) {
-            return false;
-        }
-        if (needToPushOps) {
-            pushOps(pushedOnLeft, joinBranchLeftRef, context);
-            pushOps(pushedOnRight, joinBranchRightRef, context);
-        }
-        if (intersectsAllBranches) {
-            addCondToJoin(select, join, context);
-        } else { // push down
-            Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator();
-            ILogicalExpression selectCondition = select.getCondition().getValue();
-            boolean lojToInner = false;
-            for (int j = 0; j < intersectsBranch.length; j++) {
-                Mutable<ILogicalOperator> branch = branchIter.next();
-                boolean inter = intersectsBranch[j];
-                if (inter) {
-                    if (j > 0 && isLoj) {
-                        // if a left outer join, if the select condition is not-null filtering,
-                        // we rewrite left outer join
-                        // to inner join for this case.
-                        if (containsNotNullFiltering(selectCondition)) {
-                            lojToInner = true;
-                        }
-                    }
-                    if ((j > 0 && isLoj) && containsNullFiltering(selectCondition)) {
-                        // Select is-null($$var) cannot be pushed in the right branch of a LOJ;
-                        notPushedStack.addFirst(select);
-                    } else {
-                        // Conditions for the left branch can always be pushed.
-                        // Other conditions can be pushed to the right branch of a LOJ.
-                        copySelectToBranch(select, branch, context);
-                    }
-                }
-            }
-            if (lojToInner) {
-                // Rewrites left outer join  to inner join.
-                InnerJoinOperator innerJoin = new InnerJoinOperator(join.getCondition());
-                innerJoin.getInputs().addAll(join.getInputs());
-                join = innerJoin;
-                context.computeAndSetTypeEnvironmentForOperator(join);
-            }
-        }
-        ILogicalOperator top = join;
-        for (ILogicalOperator npOp : notPushedStack) {
-            List<Mutable<ILogicalOperator>> npInpList = npOp.getInputs();
-            npInpList.clear();
-            npInpList.add(new MutableObject<ILogicalOperator>(top));
-            context.computeAndSetTypeEnvironmentForOperator(npOp);
-            top = npOp;
-        }
-        opRef.setValue(top);
-        return true;
-
-    }
-
-    private void pushOps(List<ILogicalOperator> opList, Mutable<ILogicalOperator> joinBranch,
-            IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator topOp = joinBranch.getValue();
-        ListIterator<ILogicalOperator> iter = opList.listIterator(opList.size());
-        while (iter.hasPrevious()) {
-            ILogicalOperator op = iter.previous();
-            List<Mutable<ILogicalOperator>> opInpList = op.getInputs();
-            opInpList.clear();
-            opInpList.add(new MutableObject<ILogicalOperator>(topOp));
-            topOp = op;
-            context.computeAndSetTypeEnvironmentForOperator(op);
-        }
-        joinBranch.setValue(topOp);
-    }
-
-    private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join,
-            IOptimizationContext context) {
-        ILogicalExpression cond = join.getCondition().getValue();
-        if (OperatorPropertiesUtil.isAlwaysTrueCond(cond)) { // the join was a product
-            join.getCondition().setValue(select.getCondition().getValue());
-        } else {
-            boolean bAddedToConj = false;
-            if (cond.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                AbstractFunctionCallExpression fcond = (AbstractFunctionCallExpression) cond;
-                if (fcond.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND)) {
-                    AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(context
-                            .getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND));
-                    newCond.getArguments().add(select.getCondition());
-                    newCond.getArguments().addAll(fcond.getArguments());
-                    join.getCondition().setValue(newCond);
-                    bAddedToConj = true;
-                }
-            }
-            if (!bAddedToConj) {
-                AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(context.getMetadataProvider()
-                        .lookupFunction(AlgebricksBuiltinFunctions.AND), select.getCondition(),
-                        new MutableObject<ILogicalExpression>(join.getCondition().getValue()));
-                join.getCondition().setValue(newCond);
-            }
-        }
-    }
-
-    private static void copySelectToBranch(SelectOperator select, Mutable<ILogicalOperator> branch,
-            IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator newSelect = new SelectOperator(select.getCondition(), select.getRetainNull(),
-                select.getNullPlaceholderVariable());
-        Mutable<ILogicalOperator> newRef = new MutableObject<ILogicalOperator>(branch.getValue());
-        newSelect.getInputs().add(newRef);
-        branch.setValue(newSelect);
-        context.computeAndSetTypeEnvironmentForOperator(newSelect);
-    }
-
-    /**
-     * Whether the expression contains a not-null filtering
-     * 
-     * @param expr
-     * @return true if the expression contains a not-null filtering function call; false otherwise.
-     */
-    private boolean containsNotNullFiltering(ILogicalExpression expr) {
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        ScalarFunctionCallExpression func = (ScalarFunctionCallExpression) expr;
-        if (func.getFunctionIdentifier() == AlgebricksBuiltinFunctions.AND) {
-            for (Mutable<ILogicalExpression> argumentRef : func.getArguments()) {
-                if (containsNotNullFiltering(argumentRef.getValue())) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        if (func.getFunctionIdentifier() != AlgebricksBuiltinFunctions.NOT) {
-            return false;
-        }
-        ILogicalExpression arg = func.getArguments().get(0).getValue();
-        if (arg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        ScalarFunctionCallExpression func2 = (ScalarFunctionCallExpression) arg;
-        if (func2.getFunctionIdentifier() != AlgebricksBuiltinFunctions.IS_NULL) {
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * Whether the expression contains a null filtering
-     * 
-     * @param expr
-     * @return true if the expression contains a null filtering function call; false otherwise.
-     */
-    private boolean containsNullFiltering(ILogicalExpression expr) {
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        ScalarFunctionCallExpression func = (ScalarFunctionCallExpression) expr;
-        if (func.getFunctionIdentifier() == AlgebricksBuiltinFunctions.AND) {
-            for (Mutable<ILogicalExpression> argumentRef : func.getArguments()) {
-                if (containsNullFiltering(argumentRef.getValue())) {
-                    return true;
-                }
-            }
-            return false;
-        }
-        if (func.getFunctionIdentifier() != AlgebricksBuiltinFunctions.IS_NULL) {
-            return false;
-        }
-        return true;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java
deleted file mode 100644
index 1eef9f8..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
-
- * Copyright 2009-2013 by The Regents of the University of California
-
- * Licensed under the Apache License, Version 2.0 (the "License");
-
- * you may not use this file except in compliance with the License.
-
- * you may obtain a copy of the License from
-
- * 
-
- *     http://www.apache.org/licenses/LICENSE-2.0
-
- * 
-
- * Unless required by applicable law or agreed to in writing, software
-
- * distributed under the License is distributed on an "AS IS" BASIS,
-
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-
- * See the License for the specific language governing permissions and
-
- * limitations under the License.
-
- */
-
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This rule pushes an array of subplans on top of a group-by into the
- * nested plan of the group-by.
- * 
- * @author yingyib
- */
-
-public class PushSubplanIntoGroupByRule implements IAlgebraicRewriteRule {
-    /** Stores used variables above the current operator. */
-    private final Set<LogicalVariable> usedVarsSoFar = new HashSet<LogicalVariable>();
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator parentOperator = opRef.getValue();
-        if (context.checkIfInDontApplySet(this, parentOperator)) {
-            return false;
-        }
-        context.addToDontApplySet(this, parentOperator);
-        VariableUtilities.getUsedVariables(parentOperator, usedVarsSoFar);
-        if (parentOperator.getInputs().size() <= 0) {
-            return false;
-        }
-        boolean changed = false;
-        GroupByOperator gby = null;
-        for (Mutable<ILogicalOperator> ref : parentOperator.getInputs()) {
-            AbstractLogicalOperator op = (AbstractLogicalOperator) ref.getValue();
-            /** Only processes subplan operator. */
-            List<SubplanOperator> subplans = new ArrayList<SubplanOperator>();
-            if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
-                while (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
-                    SubplanOperator currentSubplan = (SubplanOperator) op;
-                    subplans.add(currentSubplan);
-                    op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-                }
-                /** Only processes the case a group-by operator is the input of the subplan operators. */
-                if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
-                    gby = (GroupByOperator) op;
-                    List<ILogicalPlan> newGbyNestedPlans = new ArrayList<ILogicalPlan>();
-                    for (SubplanOperator subplan : subplans) {
-                        List<ILogicalPlan> subplanNestedPlans = subplan.getNestedPlans();
-                        List<ILogicalPlan> gbyNestedPlans = gby.getNestedPlans();
-                        List<ILogicalPlan> subplanNestedPlansToRemove = new ArrayList<ILogicalPlan>();
-                        for (ILogicalPlan subplanNestedPlan : subplanNestedPlans) {
-                            List<Mutable<ILogicalOperator>> rootOpRefs = subplanNestedPlan.getRoots();
-                            List<Mutable<ILogicalOperator>> rootOpRefsToRemove = new ArrayList<Mutable<ILogicalOperator>>();
-                            for (Mutable<ILogicalOperator> rootOpRef : rootOpRefs) {
-                                /** Gets free variables in the root operator of a nested plan and its descent. */
-                                Set<LogicalVariable> freeVars = new ListSet<LogicalVariable>();
-                                VariableUtilities.getUsedVariablesInDescendantsAndSelf(rootOpRef.getValue(), freeVars);
-                                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
-                                VariableUtilities.getProducedVariablesInDescendantsAndSelf(rootOpRef.getValue(),
-                                        producedVars);
-                                freeVars.removeAll(producedVars);
-                                /** * Checks whether the above freeVars are all contained in live variables * of one nested plan inside the group-by operator. * If yes, then the subplan can be pushed into the nested plan of the group-by. */
-                                for (ILogicalPlan gbyNestedPlanOriginal : gbyNestedPlans) {
-                                    // add a subplan in the original gby
-                                    if (!newGbyNestedPlans.contains(gbyNestedPlanOriginal)) {
-                                        newGbyNestedPlans.add(gbyNestedPlanOriginal);
-                                    }
-
-                                    // add a pushed subplan
-                                    ILogicalPlan gbyNestedPlan = OperatorManipulationUtil.deepCopy(
-                                            gbyNestedPlanOriginal, context);
-                                    List<Mutable<ILogicalOperator>> gbyRootOpRefs = gbyNestedPlan.getRoots();
-                                    for (int rootIndex = 0; rootIndex < gbyRootOpRefs.size(); rootIndex++) {
-                                        //set the nts for a original subplan
-                                        Mutable<ILogicalOperator> originalGbyRootOpRef = gbyNestedPlanOriginal
-                                                .getRoots().get(rootIndex);
-                                        Mutable<ILogicalOperator> originalGbyNtsRef = downToNts(originalGbyRootOpRef);
-                                        NestedTupleSourceOperator originalNts = (NestedTupleSourceOperator) originalGbyNtsRef
-                                                .getValue();
-                                        originalNts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
-
-                                        //push a new subplan if possible
-                                        Mutable<ILogicalOperator> gbyRootOpRef = gbyRootOpRefs.get(rootIndex);
-                                        Set<LogicalVariable> liveVars = new ListSet<LogicalVariable>();
-                                        VariableUtilities.getLiveVariables(gbyRootOpRef.getValue(), liveVars);
-                                        if (liveVars.containsAll(freeVars)) {
-                                            /** Does the actual push. */
-                                            Mutable<ILogicalOperator> ntsRef = downToNts(rootOpRef);
-                                            ntsRef.setValue(gbyRootOpRef.getValue());
-                                            // Removes unused vars.
-                                            AggregateOperator aggOp = (AggregateOperator) gbyRootOpRef.getValue();
-                                            for (int varIndex = aggOp.getVariables().size() - 1; varIndex >= 0; varIndex--) {
-                                                if (!freeVars.contains(aggOp.getVariables().get(varIndex))) {
-                                                    aggOp.getVariables().remove(varIndex);
-                                                    aggOp.getExpressions().remove(varIndex);
-                                                }
-                                            }
-
-                                            gbyRootOpRef.setValue(rootOpRef.getValue());
-                                            rootOpRefsToRemove.add(rootOpRef);
-
-                                            // Sets the nts for a new pushed plan.
-                                            Mutable<ILogicalOperator> oldGbyNtsRef = downToNts(gbyRootOpRef);
-                                            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) oldGbyNtsRef
-                                                    .getValue();
-                                            nts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
-
-                                            newGbyNestedPlans.add(gbyNestedPlan);
-                                            changed = true;
-                                            continue;
-                                        }
-                                    }
-                                }
-                            }
-                            rootOpRefs.removeAll(rootOpRefsToRemove);
-                            if (rootOpRefs.size() == 0) {
-                                subplanNestedPlansToRemove.add(subplanNestedPlan);
-                            }
-                        }
-                        subplanNestedPlans.removeAll(subplanNestedPlansToRemove);
-                    }
-                    if (changed) {
-                        ref.setValue(gby);
-                        gby.getNestedPlans().clear();
-                        gby.getNestedPlans().addAll(newGbyNestedPlans);
-                    }
-                }
-            }
-        }
-        if (changed) {
-            cleanup(gby);
-        }
-        return changed;
-    }
-
-    /**
-     * Removes unused aggregation variables (and expressions)
-     * 
-     * @param gby
-     * @throws AlgebricksException
-     */
-    private void cleanup(GroupByOperator gby) throws AlgebricksException {
-        for (ILogicalPlan nestedPlan : gby.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
-                AggregateOperator aggOp = (AggregateOperator) rootRef.getValue();
-                for (int varIndex = aggOp.getVariables().size() - 1; varIndex >= 0; varIndex--) {
-                    if (!usedVarsSoFar.contains(aggOp.getVariables().get(varIndex))) {
-                        aggOp.getVariables().remove(varIndex);
-                        aggOp.getExpressions().remove(varIndex);
-                    }
-                }
-            }
-
-        }
-    }
-
-    private Mutable<ILogicalOperator> downToNts(Mutable<ILogicalOperator> opRef) {
-        Mutable<ILogicalOperator> currentOpRef = opRef;
-        while (currentOpRef.getValue().getInputs().size() > 0) {
-            currentOpRef = currentOpRef.getValue().getInputs().get(0);
-        }
-        return currentOpRef;
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
deleted file mode 100644
index 9c15f77..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushSubplanWithAggregateDownThroughProductRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op1;
-        if (subplan.getNestedPlans().size() != 1) {
-            return false;
-        }
-        ILogicalPlan p = subplan.getNestedPlans().get(0);
-        if (p.getRoots().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> r = p.getRoots().get(0);
-        if (((AbstractLogicalOperator) r.getValue()).getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-
-        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
-        OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
-
-        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
-        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
-            return false;
-        }
-
-        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
-        ILogicalOperator b0 = b0Ref.getValue();
-        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(b0, b0Scm);
-        if (b0Scm.containsAll(free)) {
-            // push subplan on left branch
-            op2Ref.setValue(b0);
-            b0Ref.setValue(op1);
-            opRef.setValue(op2);
-            return true;
-        } else {
-            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
-            ILogicalOperator b1 = b1Ref.getValue();
-            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(b1, b1Scm);
-            if (b1Scm.containsAll(free)) {
-                // push subplan on right branch
-                op2Ref.setValue(b1);
-                b1Ref.setValue(op1);
-                opRef.setValue(op2);
-                return true;
-            } else {
-                return false;
-            }
-        }
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java
deleted file mode 100644
index c70fca9..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushUnnestDownThroughProductRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) {
-            return false;
-        }
-        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
-        if (join.getCondition().getValue() != ConstantExpression.TRUE) {
-            return false;
-        }
-
-        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op1, used);
-
-        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
-        ILogicalOperator b0 = b0Ref.getValue();
-        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(b0, b0Scm);
-        if (b0Scm.containsAll(used)) {
-            // push unnest on left branch
-            op2Ref.setValue(b0);
-            b0Ref.setValue(op1);
-            opRef.setValue(op2);
-            return true;
-        } else {
-            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
-            ILogicalOperator b1 = b1Ref.getValue();
-            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(b1, b1Scm);
-            if (b1Scm.containsAll(used)) {
-                // push unnest on right branch
-                op2Ref.setValue(b1);
-                b1Ref.setValue(op1);
-                opRef.setValue(op2);
-                return true;
-            } else {
-                return false;
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java
deleted file mode 100644
index b69bfb3..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * @author kereno, ecarm002, ildar.absalyamov
- *         Pushes down unnest through both branches of the union operator
- *         Before rule:
- *         ============
- *         unnest
- *         union (left_branch, right_branch, res)
- *         left_branch
- *         right_branch
- *         After rule:
- *         ============
- *         union (left_branch, right_branch, res)
- *         unnest
- *         left_branch
- *         unnest
- *         right_branch
- */
-public class PushUnnestDownThroughUnionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-
-        AbstractLogicalOperator unnest = (AbstractLogicalOperator) opRef.getValue();
-        if (unnest.getOperatorTag() != LogicalOperatorTag.UNNEST) {
-            return false;
-        }
-        UnnestOperator unnestOpRef = (UnnestOperator) opRef.getValue();
-        Mutable<ILogicalOperator> unionOp = unnest.getInputs().get(0);
-
-        AbstractLogicalOperator unionAbstractOp = (AbstractLogicalOperator) unionOp.getValue();
-        if (unionAbstractOp.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
-            return false;
-        }
-
-        LogicalVariable unnestVar1 = context.newVar();
-        UnnestOperator unnest1 = new UnnestOperator(unnestVar1, new MutableObject<ILogicalExpression>(unnestOpRef
-                .getExpressionRef().getValue().cloneExpression()));
-        LogicalVariable unnestVar2 = context.newVar();
-        UnnestOperator unnest2 = new UnnestOperator(unnestVar2, new MutableObject<ILogicalExpression>(unnestOpRef
-                .getExpressionRef().getValue().cloneExpression()));
-
-        //Getting the two topmost branched and adding them as an input to the unnests:
-        Mutable<ILogicalOperator> branch1 = unionAbstractOp.getInputs().get(0);
-        ILogicalOperator agg1 = branch1.getValue();
-        List<LogicalVariable> agg1_var = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(agg1, agg1_var);
-        Mutable<ILogicalOperator> branch2 = unionAbstractOp.getInputs().get(1);
-        ILogicalOperator agg2 = branch2.getValue();
-        List<LogicalVariable> agg2_var = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(agg2, agg2_var);
-
-        //Modifying the unnest so it has the right variable
-        List<LogicalVariable> var_unnest_1 = new ArrayList<LogicalVariable>();
-        unnest1.getExpressionRef().getValue().getUsedVariables(var_unnest_1);
-        unnest1.getExpressionRef().getValue().substituteVar(var_unnest_1.get(0), agg1_var.get(0));
-
-        List<LogicalVariable> var_unnest2 = new ArrayList<LogicalVariable>();
-        unnest2.getExpressionRef().getValue().getUsedVariables(var_unnest2);
-        unnest2.getExpressionRef().getValue().substituteVar(var_unnest2.get(0), agg2_var.get(0));
-
-        unnest1.getInputs().add(branch1);
-        unnest2.getInputs().add(branch2);
-        context.computeAndSetTypeEnvironmentForOperator(unnest1);
-        context.computeAndSetTypeEnvironmentForOperator(unnest2);
-
-        //creating a new union operator with the updated logical variables
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>(
-                1);
-        Triple<LogicalVariable, LogicalVariable, LogicalVariable> union_triple_vars = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
-                unnestVar1, unnestVar2, unnestOpRef.getVariables().get(0));
-        varMap.add(union_triple_vars);
-        UnionAllOperator unionOpFinal = new UnionAllOperator(varMap);
-
-        unionOpFinal.getInputs().add(new MutableObject<ILogicalOperator>(unnest1));
-        unionOpFinal.getInputs().add(new MutableObject<ILogicalOperator>(unnest2));
-
-        context.computeAndSetTypeEnvironmentForOperator(unionOpFinal);
-
-        opRef.setValue(unionOpFinal);
-        return true;
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
deleted file mode 100644
index aac3a18..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class ReinferAllTypesRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
-            return false;
-        }
-        typeOpRec(opRef, context);
-        return true;
-    }
-
-    private void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> r : p.getRoots()) {
-            typeOpRec(r, context);
-        }
-    }
-
-    private void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            typeOpRec(i, context);
-        }
-        if (op.hasNestedPlans()) {
-            for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
-                typePlan(p, context);
-            }
-        }
-        context.computeAndSetTypeEnvironmentForOperator(op);
-        context.addToDontApplySet(this, op);
-    }
-
-}

[41/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java
deleted file mode 100644
index 54fdf2c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class DistributeResultPOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.DISTRIBUTE_RESULT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        DistributeResultOperator write = (DistributeResultOperator) op;
-        IDataSink sink = write.getDataSink();
-        IPartitioningProperty pp = sink.getPartitioningProperty();
-        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp, null) };
-        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        DistributeResultOperator resultOp = (DistributeResultOperator) op;
-        IMetadataProvider mp = context.getMetadataProvider();
-
-        JobSpecification spec = builder.getJobSpec();
-
-        int[] columns = new int[resultOp.getExpressions().size()];
-        int i = 0;
-        for (Mutable<ILogicalExpression> exprRef : resultOp.getExpressions()) {
-            ILogicalExpression expr = exprRef.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new NotImplementedException("Only writing variable expressions is supported.");
-            }
-            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
-            LogicalVariable v = varRef.getVariableReference();
-            columns[i++] = inputSchemas[0].findVariable(v);
-        }
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
-
-        IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
-                context, columns);
-
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getResultHandleRuntime(
-                resultOp.getDataSink(), columns, pf, inputDesc, true, spec);
-
-        builder.contributeHyracksOperator(resultOp, runtimeAndConstraints.first);
-        ILogicalOperator src = resultOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, resultOp, 0);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
deleted file mode 100644
index b714979..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.LinkedList;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class EmptyTupleSourcePOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.EMPTY_TUPLE_SOURCE;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
-                new LinkedList<ILocalStructuralProperty>());
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return null;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        EmptyTupleSourceRuntimeFactory runtime = new EmptyTupleSourceRuntimeFactory();
-        RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
-        builder.contributeMicroOperator(op, runtime, recDesc);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
deleted file mode 100644
index fe438a5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.OperatorSchemaImpl;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SerializableAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor;
-
-public class ExternalGroupByPOperator extends AbstractPhysicalOperator {
-
-    private int tableSize = 0;
-    private int frameLimit = 0;
-    private List<LogicalVariable> columnSet = new ArrayList<LogicalVariable>();
-
-    public ExternalGroupByPOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList, int frameLimit,
-            int tableSize) {
-        this.tableSize = tableSize;
-        this.frameLimit = frameLimit;
-        computeColumnSet(gbyList);
-    }
-
-    public void computeColumnSet(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList) {
-        columnSet.clear();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                columnSet.add(v.getVariableReference());
-            }
-        }
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.EXTERNAL_GROUP_BY;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + columnSet;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    public List<LogicalVariable> getGbyColumns() {
-        return columnSet;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
-
-        GroupByOperator gOp = (GroupByOperator) op;
-        Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
-
-        if (!columnSet.isEmpty()) {
-            propsLocal.add(new LocalGroupingProperty(columnSet));
-        }
-        for (ILogicalPlan p : gOp.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                ILogicalOperator rOp = r.getValue();
-                propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
-            }
-        }
-
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
-        deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
-        if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
-            StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
-            pv[0] = new StructuralPropertiesVector(new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(
-                    columnSet), null), null);
-            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-        } else {
-            return emptyUnaryRequirements();
-        }
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        List<LogicalVariable> gbyCols = getGbyColumns();
-        int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]);
-        GroupByOperator gby = (GroupByOperator) op;
-        int numFds = gby.getDecorList().size();
-        int fdColumns[] = new int[numFds];
-        int j = 0;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new AlgebricksException("pre-sorted group-by expects variable references.");
-            }
-            VariableReferenceExpression v = (VariableReferenceExpression) expr;
-            LogicalVariable decor = v.getVariableReference();
-            fdColumns[j++] = inputSchemas[0].findVariable(decor);
-        }
-
-        if (gby.getNestedPlans().size() != 1) {
-            throw new AlgebricksException(
-                    "External group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        ILogicalPlan p0 = gby.getNestedPlans().get(0);
-        if (p0.getRoots().size() != 1) {
-            throw new AlgebricksException(
-                    "External group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
-        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
-
-        IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
-        List<Object> intermediateTypes = new ArrayList<Object>();
-        int n = aggOp.getExpressions().size();
-        ICopySerializableAggregateFunctionFactory[] aff = new ICopySerializableAggregateFunctionFactory[n];
-        int i = 0;
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
-        IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
-        for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
-            aff[i++] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv,
-                    inputSchemas, context);
-            intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
-                    context.getMetadataProvider()));
-        }
-
-        int[] keyAndDecFields = new int[keys.length + fdColumns.length];
-        for (i = 0; i < keys.length; ++i) {
-            keyAndDecFields[i] = keys[i];
-        }
-        for (i = 0; i < fdColumns.length; i++) {
-            keyAndDecFields[keys.length + i] = fdColumns[i];
-        }
-
-        List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList())
-            keyAndDecVariables.add(p.first);
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList())
-            keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
-
-        for (LogicalVariable var : keyAndDecVariables)
-            aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
-
-        compileSubplans(inputSchemas[0], gby, opSchema, context);
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(gbyCols,
-                aggOpInputEnv, context);
-        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        IBinaryHashFunctionFactory[] hashFunctionFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
-                gbyCols, aggOpInputEnv, context);
-
-        ICopySerializableAggregateFunctionFactory[] merges = new ICopySerializableAggregateFunctionFactory[n];
-        List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
-        IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
-        localInputSchemas[0] = new OperatorSchemaImpl();
-        for (i = 0; i < n; i++) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
-                    .get(i).getValue();
-            aggFun.getUsedVariables(usedVars);
-        }
-        i = 0;
-        for (Object type : intermediateTypes) {
-            aggOpInputEnv.setVarType(usedVars.get(i++), type);
-        }
-        for (LogicalVariable keyVar : keyAndDecVariables)
-            localInputSchemas[0].addVariable(keyVar);
-        for (LogicalVariable usedVar : usedVars)
-            localInputSchemas[0].addVariable(usedVar);
-        for (i = 0; i < n; i++) {
-            AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
-                    .get(i).getValue();
-            merges[i] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(mergeFun, aggOpInputEnv,
-                    localInputSchemas, context);
-        }
-        IAggregatorDescriptorFactory aggregatorFactory = new SerializableAggregatorDescriptorFactory(aff);
-        IAggregatorDescriptorFactory mergeFactory = new SerializableAggregatorDescriptorFactory(merges);
-
-        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
-        INormalizedKeyComputerFactory normalizedKeyFactory = JobGenHelper.variablesToAscNormalizedKeyComputerFactory(
-                gbyCols, aggOpInputEnv, context);
-        ExternalGroupOperatorDescriptor gbyOpDesc = new ExternalGroupOperatorDescriptor(spec, keyAndDecFields,
-                frameLimit, comparatorFactories, normalizedKeyFactory, aggregatorFactory, mergeFactory,
-                recordDescriptor, new HashSpillableTableFactory(tpcf, tableSize), false);
-
-        contributeOpDesc(builder, gby, gbyOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[] { 0 };
-        int[] outputDependencyLabels = new int[] { 1 };
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
deleted file mode 100644
index 1a91958..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
-
-public class HashPartitionExchangePOperator extends AbstractExchangePOperator {
-
-    private List<LogicalVariable> hashFields;
-    private INodeDomain domain;
-
-    public HashPartitionExchangePOperator(List<LogicalVariable> hashFields, INodeDomain domain) {
-        this.hashFields = hashFields;
-        this.domain = domain;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.HASH_PARTITION_EXCHANGE;
-    }
-
-    public List<LogicalVariable> getHashFields() {
-        return hashFields;
-    }
-
-    public INodeDomain getDomain() {
-        return domain;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(hashFields), domain);
-        this.deliveredProperties = new StructuralPropertiesVector(p, null);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " " + hashFields;
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        int[] keys = new int[hashFields.size()];
-        IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[hashFields.size()];
-        int i = 0;
-        IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider();
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        for (LogicalVariable v : hashFields) {
-            keys[i] = opSchema.findVariable(v);
-            hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v));
-            ++i;
-        }
-        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
-        IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
deleted file mode 100644
index fc4f116..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
-
-public class HashPartitionMergeExchangePOperator extends AbstractExchangePOperator {
-
-    private final List<OrderColumn> orderColumns;
-    private final List<LogicalVariable> partitionFields;
-    private final INodeDomain domain;
-
-    public HashPartitionMergeExchangePOperator(List<OrderColumn> orderColumns, List<LogicalVariable> partitionFields,
-            INodeDomain domain) {
-        this.orderColumns = orderColumns;
-        this.partitionFields = partitionFields;
-        this.domain = domain;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE;
-    }
-
-    public List<OrderColumn> getOrderExpressions() {
-        return orderColumns;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(partitionFields),
-                domain);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
-        List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
-        for (ILocalStructuralProperty prop : op2Locals) {
-            if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
-                locals.add(prop);
-            } else {
-                break;
-            }
-        }
-
-        this.deliveredProperties = new StructuralPropertiesVector(p, locals);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<ILocalStructuralProperty> orderProps = new LinkedList<ILocalStructuralProperty>();
-        List<OrderColumn> columns = new ArrayList<OrderColumn>();
-        for (OrderColumn oc : orderColumns) {
-            LogicalVariable var = oc.getColumn();
-            columns.add(new OrderColumn(var, oc.getOrder()));
-        }
-        orderProps.add(new LocalOrderProperty(columns));
-        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
-                orderProps) };
-        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " MERGE:" + orderColumns + " HASH:" + partitionFields;
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        int[] keys = new int[partitionFields.size()];
-        IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[partitionFields.size()];
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        {
-            int i = 0;
-            IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider();
-            for (LogicalVariable v : partitionFields) {
-                keys[i] = opSchema.findVariable(v);
-                hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v));
-                ++i;
-            }
-        }
-        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
-
-        int n = orderColumns.size();
-        int[] sortFields = new int[n];
-        IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[n];
-
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        INormalizedKeyComputerFactory nkcf = null;
-
-        int j = 0;
-        for (OrderColumn oc : orderColumns) {
-            LogicalVariable var = oc.getColumn();
-            sortFields[j] = opSchema.findVariable(var);
-            Object type = env.getVarType(var);
-            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-            comparatorFactories[j] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
-            if (j == 0 && nkcfProvider != null && type != null) {
-                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, oc.getOrder() == OrderKind.ASC);
-            }
-            j++;
-        }
-
-        IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields,
-                comparatorFactories, nkcf);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
-    }
-
-    public List<LogicalVariable> getPartitionFields() {
-        return partitionFields;
-    }
-
-    public List<OrderColumn> getOrderColumns() {
-        return orderColumns;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
deleted file mode 100644
index 3abdec9..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor;
-
-public class HybridHashJoinPOperator extends AbstractHashJoinPOperator {
-
-    private final int memSizeInFrames;
-    private final int maxInputBuildSizeInFrames;
-    private final int aveRecordsPerFrame;
-    private final double fudgeFactor;
-
-    private static final Logger LOGGER = Logger.getLogger(HybridHashJoinPOperator.class.getName());
-
-    public HybridHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
-            List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities,
-            int memSizeInFrames, int maxInputSizeInFrames, int aveRecordsPerFrame, double fudgeFactor) {
-        super(kind, partitioningType, sideLeftOfEqualities, sideRightOfEqualities);
-        this.memSizeInFrames = memSizeInFrames;
-        this.maxInputBuildSizeInFrames = maxInputSizeInFrames;
-        this.aveRecordsPerFrame = aveRecordsPerFrame;
-        this.fudgeFactor = fudgeFactor;
-
-        LOGGER.fine("HybridHashJoinPOperator constructed with: JoinKind=" + kind + ", JoinPartitioningType="
-                + partitioningType + ", List<LogicalVariable>=" + sideLeftOfEqualities + ", List<LogicalVariable>="
-                + sideRightOfEqualities + ", int memSizeInFrames=" + memSizeInFrames + ", int maxInputSize0InFrames="
-                + maxInputSizeInFrames + ", int aveRecordsPerFrame=" + aveRecordsPerFrame + ", double fudgeFactor="
-                + fudgeFactor + ".");
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.HYBRID_HASH_JOIN;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    public double getFudgeFactor() {
-        return fudgeFactor;
-    }
-
-    public int getMemSizeInFrames() {
-        return memSizeInFrames;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " " + keysLeftBranch + keysRightBranch;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
-        int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
-                keysLeftBranch, env, context);
-        IBinaryHashFunctionFamily[] hashFunFamilies = JobGenHelper.variablesToBinaryHashFunctionFamilies(
-                keysLeftBranch, env, context);
-        IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
-        int i = 0;
-        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-        for (LogicalVariable v : keysLeftBranch) {
-            Object t = env.getVarType(v);
-            comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
-        }
-
-        IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider = context
-                .getPredicateEvaluatorFactoryProvider();
-        IPredicateEvaluatorFactory predEvaluatorFactory = (predEvaluatorFactoryProvider == null ? null
-                : predEvaluatorFactoryProvider.getPredicateEvaluatorFactory(keysLeft, keysRight));
-
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
-                propagatedSchema, context);
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        IOperatorDescriptor opDesc = null;
-
-        boolean optimizedHashJoin = true;
-        for (IBinaryHashFunctionFamily family : hashFunFamilies) {
-            if (family == null) {
-                optimizedHashJoin = false;
-                break;
-            }
-        }
-
-        if (!optimizedHashJoin) {
-            try {
-                switch (kind) {
-                    case INNER: {
-                        opDesc = new HybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
-                                maxInputBuildSizeInFrames, aveRecordsPerFrame, getFudgeFactor(), keysLeft, keysRight,
-                                hashFunFactories, comparatorFactories, recDescriptor, predEvaluatorFactory);
-                        break;
-                    }
-                    case LEFT_OUTER: {
-                        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
-                        for (int j = 0; j < nullWriterFactories.length; j++) {
-                            nullWriterFactories[j] = context.getNullWriterFactory();
-                        }
-                        opDesc = new HybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
-                                maxInputBuildSizeInFrames, aveRecordsPerFrame, getFudgeFactor(), keysLeft, keysRight,
-                                hashFunFactories, comparatorFactories, recDescriptor, predEvaluatorFactory, true,
-                                nullWriterFactories);
-                        break;
-                    }
-                    default: {
-                        throw new NotImplementedException();
-                    }
-                }
-            } catch (HyracksDataException e) {
-                throw new AlgebricksException(e);
-            }
-        } else {
-            try {
-                switch (kind) {
-                    case INNER: {
-                        opDesc = new OptimizedHybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
-                                maxInputBuildSizeInFrames, getFudgeFactor(), keysLeft, keysRight, hashFunFamilies,
-                                comparatorFactories, recDescriptor, new JoinMultiComparatorFactory(comparatorFactories,
-                                        keysLeft, keysRight), new JoinMultiComparatorFactory(comparatorFactories,
-                                        keysRight, keysLeft), predEvaluatorFactory);
-                        break;
-                    }
-                    case LEFT_OUTER: {
-                        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
-                        for (int j = 0; j < nullWriterFactories.length; j++) {
-                            nullWriterFactories[j] = context.getNullWriterFactory();
-                        }
-                        opDesc = new OptimizedHybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
-                                maxInputBuildSizeInFrames, getFudgeFactor(), keysLeft, keysRight, hashFunFamilies,
-                                comparatorFactories, recDescriptor, new JoinMultiComparatorFactory(comparatorFactories,
-                                        keysLeft, keysRight), new JoinMultiComparatorFactory(comparatorFactories,
-                                        keysRight, keysLeft), predEvaluatorFactory, true, nullWriterFactories);
-                        break;
-                    }
-                    default: {
-                        throw new NotImplementedException();
-                    }
-                }
-            } catch (HyracksDataException e) {
-                throw new AlgebricksException(e);
-            }
-        }
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-
-        ILogicalOperator src1 = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getValue();
-        builder.contributeGraphEdge(src2, 0, op, 1);
-    }
-
-    @Override
-    protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        return new LinkedList<ILocalStructuralProperty>();
-    }
-
-}
-
-/**
- * {@ ITuplePairComparatorFactory} implementation for optimized hybrid hash join.
- */
-class JoinMultiComparatorFactory implements ITuplePairComparatorFactory {
-    private static final long serialVersionUID = 1L;
-
-    private final IBinaryComparatorFactory[] binaryComparatorFactories;
-    private final int[] keysLeft;
-    private final int[] keysRight;
-
-    public JoinMultiComparatorFactory(IBinaryComparatorFactory[] binaryComparatorFactory, int[] keysLeft,
-            int[] keysRight) {
-        this.binaryComparatorFactories = binaryComparatorFactory;
-        this.keysLeft = keysLeft;
-        this.keysRight = keysRight;
-    }
-
-    @Override
-    public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
-        IBinaryComparator[] binaryComparators = new IBinaryComparator[binaryComparatorFactories.length];
-        for (int i = 0; i < binaryComparators.length; i++) {
-            binaryComparators[i] = binaryComparatorFactories[i].createBinaryComparator();
-        }
-        return new JoinMultiComparator(binaryComparators, keysLeft, keysRight);
-    }
-}
-
-/**
- * {@ ITuplePairComparator} implementation for optimized hybrid hash join.
- * The comparator applies multiple binary comparators, one for each key pairs
- */
-class JoinMultiComparator implements ITuplePairComparator {
-    private final IBinaryComparator[] binaryComparators;
-    private final int[] keysLeft;
-    private final int[] keysRight;
-
-    public JoinMultiComparator(IBinaryComparator[] bComparator, int[] keysLeft, int[] keysRight) {
-        this.binaryComparators = bComparator;
-        this.keysLeft = keysLeft;
-        this.keysRight = keysRight;
-    }
-
-    @Override
-    public int compare(IFrameTupleAccessor accessor0, int tIndex0, IFrameTupleAccessor accessor1, int tIndex1)
-            throws HyracksDataException {
-        int tStart0 = accessor0.getTupleStartOffset(tIndex0);
-        int fStartOffset0 = accessor0.getFieldSlotsLength() + tStart0;
-
-        int tStart1 = accessor1.getTupleStartOffset(tIndex1);
-        int fStartOffset1 = accessor1.getFieldSlotsLength() + tStart1;
-
-        for (int i = 0; i < binaryComparators.length; i++) {
-            int fStart0 = accessor0.getFieldStartOffset(tIndex0, keysLeft[i]);
-            int fEnd0 = accessor0.getFieldEndOffset(tIndex0, keysLeft[i]);
-            int fLen0 = fEnd0 - fStart0;
-
-            int fStart1 = accessor1.getFieldStartOffset(tIndex1, keysRight[i]);
-            int fEnd1 = accessor1.getFieldEndOffset(tIndex1, keysRight[i]);
-            int fLen1 = fEnd1 - fStart1;
-
-            int c = binaryComparators[i].compare(accessor0.getBuffer().array(), fStart0 + fStartOffset0, fLen0,
-                    accessor1.getBuffer().array(), fStart1 + fStartOffset1, fLen1);
-            if (c != 0) {
-                return c;
-            }
-        }
-        return 0;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
deleted file mode 100644
index 55616f9..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
-
-public class InMemoryHashJoinPOperator extends AbstractHashJoinPOperator {
-
-    private final int tableSize;
-
-    /**
-     * builds on the first operator and probes on the second.
-     */
-
-    public InMemoryHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
-            List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities, int tableSize) {
-        super(kind, partitioningType, sideLeftOfEqualities, sideRightOfEqualities);
-        this.tableSize = tableSize;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.IN_MEMORY_HASH_JOIN;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " " + keysLeftBranch + keysRightBranch;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
-        int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
-                keysLeftBranch, env, context);
-        IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
-        int i = 0;
-        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-        for (LogicalVariable v : keysLeftBranch) {
-            Object t = env.getVarType(v);
-            comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
-        }
-        
-        IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider = context.getPredicateEvaluatorFactoryProvider();
-        IPredicateEvaluatorFactory predEvaluatorFactory = ( predEvaluatorFactoryProvider == null ? null : predEvaluatorFactoryProvider.getPredicateEvaluatorFactory(keysLeft, keysRight));
-        
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
-                propagatedSchema, context);
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        IOperatorDescriptor opDesc = null;
-
-        switch (kind) {
-            case INNER: {
-                opDesc = new InMemoryHashJoinOperatorDescriptor(spec, keysLeft, keysRight, hashFunFactories,
-                        comparatorFactories, recDescriptor, tableSize, predEvaluatorFactory);
-                break;
-            }
-            case LEFT_OUTER: {
-                INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
-                for (int j = 0; j < nullWriterFactories.length; j++) {
-                    nullWriterFactories[j] = context.getNullWriterFactory();
-                }
-                opDesc = new InMemoryHashJoinOperatorDescriptor(spec, keysLeft, keysRight, hashFunFactories,
-                        comparatorFactories, predEvaluatorFactory, recDescriptor, true, nullWriterFactories, tableSize);
-                break;
-            }
-            default: {
-                throw new NotImplementedException();
-            }
-        }
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-
-        ILogicalOperator src1 = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getValue();
-        builder.contributeGraphEdge(src2, 0, op, 1);
-    }
-
-    @Override
-    protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
-        List<ILocalStructuralProperty> lp0 = pv0.getLocalProperties();
-        if (lp0 != null) {
-            // maintains the local properties on the probe side
-            return new LinkedList<ILocalStructuralProperty>(lp0);
-        }
-        return new LinkedList<ILocalStructuralProperty>();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
deleted file mode 100644
index 4a7d875..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.sort.InMemorySortRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class InMemoryStableSortPOperator extends AbstractStableSortPOperator {
-
-    public InMemoryStableSortPOperator() {
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.IN_MEMORY_STABLE_SORT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        int n = sortColumns.length;
-        int[] sortFields = new int[n];
-        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
-        int i = 0;
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        INormalizedKeyComputerFactory nkcf = null;
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        for (OrderColumn oc : sortColumns) {
-            LogicalVariable var = oc.getColumn();
-            sortFields[i] = opSchema.findVariable(var);
-            Object type = env.getVarType(var);
-            OrderKind order = oc.getOrder();
-            if (i == 0 && nkcfProvider != null && type != null) {
-                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
-            }
-
-            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
-            i++;
-        }
-
-        IPushRuntimeFactory runtime = new InMemorySortRuntimeFactory(sortFields, nkcf, comps, null);
-        builder.contributeMicroOperator(op, runtime, recDescriptor);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
deleted file mode 100644
index db00b9e..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class IndexBulkloadPOperator extends AbstractPhysicalOperator {
-
-    private final List<LogicalVariable> primaryKeys;
-    private final List<LogicalVariable> secondaryKeys;
-    private final List<LogicalVariable> additionalFilteringKeys;
-    private final Mutable<ILogicalExpression> filterExpr;
-    private final IDataSourceIndex<?, ?> dataSourceIndex;
-
-    public IndexBulkloadPOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            List<LogicalVariable> additionalFilteringKeys, Mutable<ILogicalExpression> filterExpr,
-            IDataSourceIndex<?, ?> dataSourceIndex) {
-        this.primaryKeys = primaryKeys;
-        this.secondaryKeys = secondaryKeys;
-        this.additionalFilteringKeys = additionalFilteringKeys;
-        this.filterExpr = filterExpr;
-        this.dataSourceIndex = dataSourceIndex;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.INDEX_BULKLOAD;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<LogicalVariable> scanVariables = new ArrayList<>();
-        scanVariables.addAll(primaryKeys);
-        scanVariables.add(new LogicalVariable(-1));
-        IPhysicalPropertiesVector physicalProps = dataSourceIndex.getDataSource().getPropertiesProvider()
-                .computePropertiesVector(scanVariables);
-        List<ILocalStructuralProperty> localProperties = new ArrayList<>();
-        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-        // Data needs to be sorted based on the [token, number of token, PK]
-        // OR [token, PK] if the index is not partitioned
-        for (LogicalVariable skVar : secondaryKeys) {
-            orderColumns.add(new OrderColumn(skVar, OrderKind.ASC));
-        }
-        for (LogicalVariable pkVar : primaryKeys) {
-            orderColumns.add(new OrderColumn(pkVar, OrderKind.ASC));
-        }
-        localProperties.add(new LocalOrderProperty(orderColumns));
-        StructuralPropertiesVector spv = new StructuralPropertiesVector(physicalProps.getPartitioningProperty(),
-                localProperties);
-        return new PhysicalRequirements(new IPhysicalPropertiesVector[] { spv },
-                IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        IndexInsertDeleteOperator indexInsertDeleteOp = (IndexInsertDeleteOperator) op;
-        assert indexInsertDeleteOp.getOperation() == Kind.INSERT;
-        assert indexInsertDeleteOp.isBulkload();
-
-        IMetadataProvider mp = context.getMetadataProvider();
-        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
-        JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getIndexInsertRuntime(
-                dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys,
-                additionalFilteringKeys, null, inputDesc, context, spec, true);
-        builder.contributeHyracksOperator(indexInsertDeleteOp, runtimeAndConstraints.first);
-        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = indexInsertDeleteOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, indexInsertDeleteOp, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
deleted file mode 100644
index 4bc4c06..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class IndexInsertDeletePOperator extends AbstractPhysicalOperator {
-
-    private final List<LogicalVariable> primaryKeys;
-    private final List<LogicalVariable> secondaryKeys;
-    private final ILogicalExpression filterExpr;
-    private final IDataSourceIndex<?, ?> dataSourceIndex;
-    private final List<LogicalVariable> additionalFilteringKeys;
-
-    public IndexInsertDeletePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            List<LogicalVariable> additionalFilteringKeys, Mutable<ILogicalExpression> filterExpr,
-            IDataSourceIndex<?, ?> dataSourceIndex) {
-        this.primaryKeys = primaryKeys;
-        this.secondaryKeys = secondaryKeys;
-        if (filterExpr != null) {
-            this.filterExpr = filterExpr.getValue();
-        } else {
-            this.filterExpr = null;
-        }
-        this.dataSourceIndex = dataSourceIndex;
-        this.additionalFilteringKeys = additionalFilteringKeys;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.INDEX_INSERT_DELETE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
-        scanVariables.addAll(primaryKeys);
-        scanVariables.add(new LogicalVariable(-1));
-        IPhysicalPropertiesVector r = dataSourceIndex.getDataSource().getPropertiesProvider()
-                .computePropertiesVector(scanVariables);
-        r.getLocalProperties().clear();
-        IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
-        requirements[0] = r;
-        return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        IndexInsertDeleteOperator insertDeleteOp = (IndexInsertDeleteOperator) op;
-        IMetadataProvider mp = context.getMetadataProvider();
-
-        JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
-
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
-        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(insertDeleteOp);
-        if (insertDeleteOp.getOperation() == Kind.INSERT) {
-            runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
-                    primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec, false);
-        } else {
-            runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
-                    primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec);
-        }
-        builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
-        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

[33/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
new file mode 100644
index 0000000..7f812f3
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+import java.io.Serializable;
+
+public final class FunctionIdentifier implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private final String namespace;
+    private final String name;
+    private final int arity;
+
+    public final static int VARARGS = -1;
+
+    public FunctionIdentifier(String namespace, String name) {
+        this(namespace, name, VARARGS);
+    }
+
+    public FunctionIdentifier(String namespace, String name, int arity) {
+        this.namespace = namespace;
+        this.name = name;
+        this.arity = arity;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (super.equals(o)) {
+            return true;
+        }
+        if (o instanceof FunctionIdentifier) {
+            FunctionIdentifier ofi = (FunctionIdentifier) o;
+            return ofi.getNamespace().equals(getNamespace()) && ofi.name.equals(name);
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return name.hashCode() + namespace.hashCode();
+    }
+
+    public String toString() {
+        return getNamespace() + ":" + name;
+    }
+
+    public int getArity() {
+        return arity;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
new file mode 100644
index 0000000..02cd6de
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+public interface IFunctionInfo {
+    FunctionIdentifier getFunctionIdentifier();
+
+    public boolean isFunctional();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSink.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSink.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSink.java
new file mode 100644
index 0000000..a7b7bc5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSink.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+
+public interface IDataSink {
+    public Object getId();
+
+    public Object[] getSchemaTypes();
+
+    public IPartitioningProperty getPartitioningProperty();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java
new file mode 100644
index 0000000..ed5cb4b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+
+public interface IDataSource<T> {
+    public T getId();
+
+    public Object[] getSchemaTypes();
+
+    public IDataSourcePropertiesProvider getPropertiesProvider();
+
+    public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
new file mode 100644
index 0000000..20a2dcf
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+public interface IDataSourceIndex<I, T> {
+    public I getId();
+
+    public IDataSource<T> getDataSource();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
new file mode 100644
index 0000000..3ae1e31
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+
+public interface IDataSourcePropertiesProvider {
+    public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
new file mode 100644
index 0000000..c71b352
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -0,0 +1,192 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public interface IMetadataProvider<S, I> {
+    public IDataSource<S> findDataSource(S id) throws AlgebricksException;
+
+    /**
+     * Obs: A scanner may choose to contribute a null
+     * AlgebricksPartitionConstraint and implement
+     * contributeSchedulingConstraints instead.
+     */
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
+            List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+            List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+            IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+            throws AlgebricksException;
+
+    public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource);
+
+    public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
+            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
+            throws AlgebricksException;
+
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
+            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
+            JobSpecification spec) throws AlgebricksException;
+
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> dataSource,
+            IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
+            List<LogicalVariable> additionalNonKeyFields, JobGenContext context, JobSpecification jobSpec)
+            throws AlgebricksException;
+
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
+            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification jobSpec, boolean bulkload) throws AlgebricksException;
+
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
+            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+
+    /**
+     * Creates the insert runtime of IndexInsertDeletePOperator, which models
+     * insert/delete operations into a secondary index.
+     * 
+     * @param dataSource
+     *            Target secondary index.
+     * @param propagatedSchema
+     *            Output schema of the insert/delete operator to be created.
+     * @param inputSchemas
+     *            Output schemas of the insert/delete operator to be created.
+     * @param typeEnv
+     *            Type environment of the original IndexInsertDeleteOperator operator.
+     * @param primaryKeys
+     *            Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+     * @param secondaryKeys
+     *            Variables for the secondary-index keys.
+     * @param additionalNonKeyFields
+     *            Additional variables that can be passed to the secondary index as payload.
+     *            This can be useful when creating a second filter on a non-primary and non-secondary
+     *            fields for additional pruning power.
+     * @param filterExpr
+     *            Filtering expression to be pushed inside the runtime op.
+     *            Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+     * @param recordDesc
+     *            Output record descriptor of the runtime op to be created.
+     * @param context
+     *            Job generation context.
+     * @param spec
+     *            Target job specification.
+     * @return
+     *         A Hyracks IOperatorDescriptor and its partition constraint.
+     * @throws AlgebricksException
+     */
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
+            IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException;
+
+    /**
+     * Creates the delete runtime of IndexInsertDeletePOperator, which models
+     * insert/delete operations into a secondary index.
+     * 
+     * @param dataSource
+     *            Target secondary index.
+     * @param propagatedSchema
+     *            Output schema of the insert/delete operator to be created.
+     * @param inputSchemas
+     *            Output schemas of the insert/delete operator to be created.
+     * @param typeEnv
+     *            Type environment of the original IndexInsertDeleteOperator operator.
+     * @param primaryKeys
+     *            Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+     * @param secondaryKeys
+     *            Variables for the secondary-index keys.
+     * @param additionalNonKeyFields
+     *            Additional variables that can be passed to the secondary index as payload.
+     *            This can be useful when creating a second filter on a non-primary and non-secondary
+     *            fields for additional pruning power.
+     * @param filterExpr
+     *            Filtering expression to be pushed inside the runtime op.
+     *            Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+     * @param recordDesc
+     *            Output record descriptor of the runtime op to be created.
+     * @param context
+     *            Job generation context.
+     * @param spec
+     *            Target job specification.
+     * @return
+     *         A Hyracks IOperatorDescriptor and its partition constraint.
+     * @throws AlgebricksException
+     */
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
+            IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification spec) throws AlgebricksException;
+
+    /**
+     * Creates the TokenizeOperator for IndexInsertDeletePOperator, which tokenizes
+     * secondary key into [token, number of token] pair in a length-partitioned index.
+     * In case of non length-partitioned index, it tokenizes secondary key into [token].
+     * 
+     * @param dataSource
+     *            Target secondary index.
+     * @param propagatedSchema
+     *            Output schema of the insert/delete operator to be created.
+     * @param inputSchemas
+     *            Output schemas of the insert/delete operator to be created.
+     * @param typeEnv
+     *            Type environment of the original IndexInsertDeleteOperator operator.
+     * @param primaryKeys
+     *            Variables for the dataset's primary keys that the dataSource secondary index belongs to.
+     * @param secondaryKeys
+     *            Variables for the secondary-index keys.
+     * @param filterExpr
+     *            Filtering expression to be pushed inside the runtime op.
+     *            Such a filter may, e.g., exclude NULLs from being inserted/deleted.
+     * @param recordDesc
+     *            Output record descriptor of the runtime op to be created.
+     * @param context
+     *            Job generation context.
+     * @param spec
+     *            Target job specification.
+     * @return
+     *         A Hyracks IOperatorDescriptor and its partition constraint.
+     * @throws AlgebricksException
+     */
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(
+            IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec,
+            boolean bulkload) throws AlgebricksException;
+
+    public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
+
+    public IFunctionInfo lookupFunction(FunctionIdentifier fid);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
new file mode 100644
index 0000000..687025a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+
+/**
+ * @author Nicola
+ */
+public abstract class AbstractAssignOperator extends AbstractLogicalOperator {
+    protected final List<LogicalVariable> variables;
+    protected final List<Mutable<ILogicalExpression>> expressions;
+
+    public AbstractAssignOperator() {
+        this.variables = new ArrayList<LogicalVariable>();
+        this.expressions = new ArrayList<Mutable<ILogicalExpression>>();
+    }
+
+    public AbstractAssignOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+        this.variables = variables;
+        this.expressions = expressions;
+    }
+
+    public List<LogicalVariable> getVariables() {
+        return variables;
+    }
+
+    public List<Mutable<ILogicalExpression>> getExpressions() {
+        return expressions;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+        schema.addAll(variables);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean modif = false;
+        for (int i = 0; i < expressions.size(); i++) {
+            if (visitor.transform(expressions.get(i))) {
+                modif = true;
+            }
+        }
+        return modif;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
new file mode 100644
index 0000000..6c00412
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+
+public abstract class AbstractBinaryJoinOperator extends AbstractLogicalOperator {
+    protected final Mutable<ILogicalExpression> condition;
+    protected JoinKind joinKind;
+
+    public enum JoinKind {
+        INNER,
+        LEFT_OUTER
+    }
+
+    public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition) {
+        this.joinKind = joinKind;
+        this.condition = condition;
+    }
+
+    public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition,
+            Mutable<ILogicalOperator> input1, Mutable<ILogicalOperator> input2) {
+        this(joinKind, condition);
+        inputs.add(input1);
+        inputs.add(input2);
+    }
+
+    public Mutable<ILogicalExpression> getCondition() {
+        return condition;
+    }
+
+    public JoinKind getJoinKind() {
+        return joinKind;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+        schema.addAll(inputs.get(1).getValue().getSchema());
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.concat(VariablePropagationPolicy.ALL, VariablePropagationPolicy.ALL);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return visitor.transform(condition);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java
new file mode 100644
index 0000000..85660e2
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+
+import java.util.List;
+
+public abstract class AbstractDataSourceOperator extends AbstractScanOperator {
+    protected IDataSource<?> dataSource;
+
+    public AbstractDataSourceOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
+        super(variables);
+        this.dataSource = dataSource;
+    }
+
+    public IDataSource<?> getDataSource() {
+        return dataSource;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
new file mode 100644
index 0000000..2b3c60a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+
+/**
+ * @author rico
+ */
+public abstract class AbstractExtensibleLogicalOperator implements IOperatorExtension {
+
+    private AbstractLogicalOperator.ExecutionMode mode = AbstractLogicalOperator.ExecutionMode.UNPARTITIONED;
+    protected List<LogicalVariable> schema;
+    protected IPhysicalOperator physicalOperator;
+
+    @Override
+    public ExecutionMode getExecutionMode() {
+        return mode;
+    }
+
+    @Override
+    public void setExecutionMode(ExecutionMode mode) {
+        this.mode = mode;
+    }
+
+    @Override
+    public void setSchema(List<LogicalVariable> schema) {
+        this.schema = schema;
+    }
+
+    @Override
+    public IPhysicalOperator getPhysicalOperator() {
+        return physicalOperator;
+    }
+
+    @Override
+    public void setPhysicalOperator(IPhysicalOperator physicalOperator) {
+        this.physicalOperator = physicalOperator;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
new file mode 100644
index 0000000..528ae4e
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+
+public abstract class AbstractLogicalOperator implements ILogicalOperator {
+
+    /*********************************************************************
+     * UNPARTITIONED, the input data is not partitioned
+     * PARTITIONED, the input data is partitioned, the operator is executed on
+     * each partition and may receive input from other partitions (e.g. if it is
+     * a join or an aggregate)
+     * LOCAL, the input data is partitioned, the operator is executed on each
+     * partition and only processes data from that partition
+     */
+
+    public static enum ExecutionMode {
+        UNPARTITIONED,
+        PARTITIONED,
+        LOCAL
+    }
+
+    private AbstractLogicalOperator.ExecutionMode mode = AbstractLogicalOperator.ExecutionMode.UNPARTITIONED;
+    protected IPhysicalOperator physicalOperator;
+    private final Map<String, Object> annotations = new HashMap<String, Object>();
+    private boolean bJobGenEnabled = true;
+
+    final protected List<Mutable<ILogicalOperator>> inputs;
+    // protected List<LogicalOperatorReference> outputs;
+    protected List<LogicalVariable> schema;
+
+    public AbstractLogicalOperator() {
+        inputs = new ArrayList<Mutable<ILogicalOperator>>();
+        // outputs = new ArrayList<LogicalOperatorReference>();
+    }
+
+    @Override
+    public abstract LogicalOperatorTag getOperatorTag();
+
+    public ExecutionMode getExecutionMode() {
+        return mode;
+    }
+
+    public void setExecutionMode(ExecutionMode mode) {
+        this.mode = mode;
+    }
+
+    @Override
+    public List<LogicalVariable> getSchema() {
+        return schema;
+    }
+
+    public void setPhysicalOperator(IPhysicalOperator physicalOp) {
+        this.physicalOperator = physicalOp;
+    }
+
+    public IPhysicalOperator getPhysicalOperator() {
+        return physicalOperator;
+    }
+
+    /**
+     * @return for each child, one vector of required physical properties
+     */
+
+    @Override
+    public final PhysicalRequirements getRequiredPhysicalPropertiesForChildren(
+            IPhysicalPropertiesVector requiredProperties) {
+        return physicalOperator.getRequiredPropertiesForChildren(this, requiredProperties);
+    }
+
+    /**
+     * @return the physical properties that this operator delivers, based on
+     *         what its children deliver
+     */
+
+    @Override
+    public final IPhysicalPropertiesVector getDeliveredPhysicalProperties() {
+        return physicalOperator.getDeliveredProperties();
+    }
+
+    @Override
+    public final void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException {
+        physicalOperator.computeDeliveredProperties(this, context);
+    }
+
+    @Override
+    public final List<Mutable<ILogicalOperator>> getInputs() {
+        return inputs;
+    }
+
+    // @Override
+    // public final List<LogicalOperatorReference> getOutputs() {
+    // return outputs;
+    // }
+
+    @Override
+    public final boolean hasInputs() {
+        return !inputs.isEmpty();
+    }
+
+    public boolean hasNestedPlans() {
+        return false;
+    }
+
+    @Override
+    public Map<String, Object> getAnnotations() {
+        return annotations;
+    }
+
+    @Override
+    public void removeAnnotation(String annotationName) {
+        annotations.remove(annotationName);
+    }
+
+    @Override
+    public final void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        if (bJobGenEnabled) {
+            if (physicalOperator == null) {
+                throw new AlgebricksException("Physical operator not set for operator: " + this);
+            }
+            physicalOperator.contributeRuntimeOperator(builder, context, this, propagatedSchema, inputSchemas,
+                    outerPlanSchema);
+        }
+    }
+
+    public void disableJobGen() {
+        bJobGenEnabled = false;
+    }
+
+    public boolean isJobGenEnabled() {
+        return bJobGenEnabled;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    protected PropagatingTypeEnvironment createPropagatingAllInputsTypeEnvironment(ITypingContext ctx) {
+        int n = inputs.size();
+        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
+        for (int i = 0; i < n; i++) {
+            envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
+        }
+        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+                ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+    }
+
+    @Override
+    public boolean requiresVariableReferenceExpressions() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
new file mode 100644
index 0000000..417c959
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public abstract class AbstractOperatorWithNestedPlans extends AbstractLogicalOperator {
+    protected final List<ILogicalPlan> nestedPlans;
+
+    public AbstractOperatorWithNestedPlans() {
+        nestedPlans = new ArrayList<ILogicalPlan>();
+    }
+
+    public AbstractOperatorWithNestedPlans(List<ILogicalPlan> nestedPlans) {
+        this.nestedPlans = nestedPlans;
+    }
+
+    public List<ILogicalPlan> getNestedPlans() {
+        return nestedPlans;
+    }
+
+    @Override
+    public boolean hasNestedPlans() {
+        return true;
+    }
+
+    public LinkedList<Mutable<ILogicalOperator>> allRootsInReverseOrder() {
+        LinkedList<Mutable<ILogicalOperator>> allRoots = new LinkedList<Mutable<ILogicalOperator>>();
+        for (ILogicalPlan p : nestedPlans) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                allRoots.addFirst(r);
+            }
+        }
+        return allRoots;
+    }
+
+    //
+    // @Override
+    // public void computeConstraintsAndEquivClasses() {
+    // for (ILogicalPlan p : nestedPlans) {
+    // for (LogicalOperatorReference r : p.getRoots()) {
+    // AbstractLogicalOperator op = (AbstractLogicalOperator) r.getOperator();
+    // equivalenceClasses.putAll(op.getEquivalenceClasses());
+    // functionalDependencies.addAll(op.getFDs());
+    // }
+    // }
+    // }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+        for (ILogicalPlan p : nestedPlans) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                schema.addAll(r.getValue().getSchema());
+            }
+        }
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    public abstract void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars);
+
+    public abstract void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
new file mode 100644
index 0000000..e18fbfb
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+
+public abstract class AbstractScanOperator extends AbstractLogicalOperator {
+    protected List<LogicalVariable> variables;
+
+    public AbstractScanOperator(List<LogicalVariable> variables) {
+        this.variables = variables;
+    }
+
+    public List<LogicalVariable> getVariables() {
+        return variables;
+    }
+
+    public void setVariables(List<LogicalVariable> variables) {
+        this.variables = variables;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+        schema.addAll(variables);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (sources.length > 0) {
+                    target.addAllVariables(sources[0]);
+                }
+                for (LogicalVariable v : variables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
new file mode 100644
index 0000000..fdc827f
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+
+public abstract class AbstractUnnestOperator extends AbstractScanOperator {
+
+    protected final Mutable<ILogicalExpression> expression;
+
+    public AbstractUnnestOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression) {
+        super(variables);
+        this.expression = expression;
+    }
+
+    public Mutable<ILogicalExpression> getExpressionRef() {
+        return expression;
+    }
+
+    @Override
+    public boolean isMap() {
+        return true;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return visitor.transform(expression);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
new file mode 100644
index 0000000..371da4c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class AggregateOperator extends AbstractAssignOperator {
+
+    // private ArrayList<AggregateFunctionCallExpression> expressions;
+    // TODO type safe list of expressions
+    private List<Mutable<ILogicalExpression>> mergeExpressions;
+    private boolean global;
+
+    public AggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+        super(variables, expressions);
+        global = true;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.AGGREGATE;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitAggregateOperator(this, arg);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                for (LogicalVariable v : variables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(variables);
+    }
+
+    public void setMergeExpressions(List<Mutable<ILogicalExpression>> merges) {
+        mergeExpressions = merges;
+    }
+
+    public List<Mutable<ILogicalExpression>> getMergeExpressions() {
+        return mergeExpressions;
+    }
+
+    public void setGlobal(boolean global) {
+        this.global = global;
+    }
+
+    public boolean isGlobal() {
+        return global;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getMetadataProvider());
+        IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            Object t = ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(),
+                    ctx.getMetadataProvider(), env2);
+            env.setVarType(variables.get(i), t);
+        }
+        return env;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
new file mode 100644
index 0000000..9a8c428
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+/**
+ * It corresponds to the Map operator in other algebras.
+ * 
+ * @author Nicola
+ */
+
+public class AssignOperator extends AbstractAssignOperator {
+	
+	private LocalOrderProperty explicitOrderingProperty;
+
+    public AssignOperator(List<LogicalVariable> vars, List<Mutable<ILogicalExpression>> exprs) {
+        super(vars, exprs);
+    }
+
+    public AssignOperator(LogicalVariable var, Mutable<ILogicalExpression> expr) {
+        super();
+        this.variables.add(var);
+        this.expressions.add(expr);
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.ASSIGN;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitAssignOperator(this, arg);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                target.addAllVariables(sources[0]);
+                for (LogicalVariable v : variables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+
+    }
+
+    @Override
+    public boolean isMap() {
+        return true;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        PropagatingTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            env.setVarType(
+                    variables.get(i),
+                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
+                            env));
+            if (expressions.get(i).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                LogicalVariable var = ((VariableReferenceExpression) expressions.get(i).getValue())
+                        .getVariableReference();
+                for (List<LogicalVariable> list : env.getCorrelatedNullableVariableLists()) {
+                    if (list.contains(var)) {
+                        list.add(variables.get(i));
+                    }
+                }
+            }
+        }
+        return env;
+    }
+
+    @Override
+    public boolean requiresVariableReferenceExpressions() {
+        return false;
+    }
+
+    public LocalOrderProperty getExplicitOrderingProperty() {
+		return explicitOrderingProperty;
+	}
+
+	public void setExplicitOrderingProperty(
+			LocalOrderProperty explicitOrderingProperty) {
+		this.explicitOrderingProperty = explicitOrderingProperty;
+	}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
new file mode 100644
index 0000000..ff21478
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class DataSourceScanOperator extends AbstractDataSourceOperator {
+    private List<LogicalVariable> projectVars;
+
+    private boolean projectPushed = false;
+
+    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+    private List<LogicalVariable> minFilterVars;
+    private List<LogicalVariable> maxFilterVars;
+
+    public DataSourceScanOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
+        super(variables, dataSource);
+        projectVars = new ArrayList<LogicalVariable>();
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.DATASOURCESCAN;
+    }
+
+    @Override
+    public <R, S> R accept(ILogicalOperatorVisitor<R, S> visitor, S arg) throws AlgebricksException {
+        return visitor.visitDataScanOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    public void addProjectVariables(Collection<LogicalVariable> vars) {
+        projectVars.addAll(vars);
+        projectPushed = true;
+    }
+
+    public List<LogicalVariable> getProjectVariables() {
+        return projectVars;
+    }
+
+    public boolean isProjectPushed() {
+        return projectPushed;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (sources.length > 0) {
+                    target.addAllVariables(sources[0]);
+                }
+                List<LogicalVariable> outputVariables = projectPushed ? projectVars : variables;
+                for (LogicalVariable v : outputVariables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+        Object[] types = dataSource.getSchemaTypes();
+        int i = 0;
+        for (LogicalVariable v : variables) {
+            env.setVarType(v, types[i]);
+            ++i;
+        }
+        return env;
+    }
+
+    public List<LogicalVariable> getMinFilterVars() {
+        return minFilterVars;
+    }
+
+    public void setMinFilterVars(List<LogicalVariable> minFilterVars) {
+        this.minFilterVars = minFilterVars;
+    }
+
+    public List<LogicalVariable> getMaxFilterVars() {
+        return maxFilterVars;
+    }
+
+    public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) {
+        this.maxFilterVars = maxFilterVars;
+    }
+
+    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+        this.additionalFilteringExpressions = additionalFilteringExpressions;
+    }
+
+    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+        return additionalFilteringExpressions;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
new file mode 100644
index 0000000..3d8c4b6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class DistinctOperator extends AbstractLogicalOperator {
+    private final List<Mutable<ILogicalExpression>> expressions;
+
+    public DistinctOperator(List<Mutable<ILogicalExpression>> expressions) {
+        this.expressions = expressions;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.DISTINCT;
+    }
+
+    public List<Mutable<ILogicalExpression>> getExpressions() {
+        return expressions;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(this.getDistinctByVarList());
+        List<LogicalVariable> inputSchema = inputs.get(0).getValue().getSchema();
+        for (LogicalVariable var : inputSchema) {
+            if (!schema.contains(var)) {
+                schema.add(var);
+            }
+        }
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                /** make sure distinct key vars laid-out first */
+                for (LogicalVariable keyVar : getDistinctByVarList()) {
+                    target.addVariable(keyVar);
+                }
+                /** add other source vars */
+                for (IOperatorSchema srcSchema : sources) {
+                    for (LogicalVariable srcVar : srcSchema)
+                        if (target.findVariable(srcVar) < 0) {
+                            target.addVariable(srcVar);
+                        }
+                }
+            }
+        };
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean changed = false;
+        for (Mutable<ILogicalExpression> e : expressions) {
+            if (visitor.transform(e)) {
+                changed = true;
+            }
+        }
+        return changed;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitDistinctOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    public List<LogicalVariable> getDistinctByVarList() {
+        List<LogicalVariable> varList = new ArrayList<LogicalVariable>(expressions.size());
+        for (Mutable<ILogicalExpression> eRef : expressions) {
+            ILogicalExpression e = eRef.getValue();
+            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) e;
+                varList.add(v.getVariableReference());
+            }
+        }
+        return varList;
+    }
+
+    public boolean isDistinctByVar(LogicalVariable var) {
+        for (Mutable<ILogicalExpression> eRef : expressions) {
+            ILogicalExpression e = eRef.getValue();
+            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) e;
+                if (v.getVariableReference() == var) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java
new file mode 100644
index 0000000..3e28d73
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class DistributeResultOperator extends AbstractLogicalOperator {
+    private List<Mutable<ILogicalExpression>> expressions;
+    private IDataSink dataSink;
+
+    public DistributeResultOperator(List<Mutable<ILogicalExpression>> expressions, IDataSink dataSink) {
+        this.expressions = expressions;
+        this.dataSink = dataSink;
+    }
+
+    public List<Mutable<ILogicalExpression>> getExpressions() {
+        return expressions;
+    }
+
+    public IDataSink getDataSink() {
+        return dataSink;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.DISTRIBUTE_RESULT;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitDistributeResultOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean modif = false;
+        for (int i = 0; i < expressions.size(); i++) {
+            boolean b = visitor.transform(expressions.get(i));
+            if (b) {
+                modif = true;
+            }
+        }
+        return modif;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return false; // actually depends on the physical op.
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
new file mode 100644
index 0000000..7d16ed1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class EmptyTupleSourceOperator extends AbstractLogicalOperator {
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.EMPTYTUPLESOURCE;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.NONE;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+        // do nothing
+        return false;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitEmptyTupleSourceOperator(this, arg);
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException {
+        return new IVariableTypeEnvironment() {
+
+            @Override
+            public void setVarType(LogicalVariable var, Object type) {
+                throw new IllegalStateException();
+            }
+
+            @Override
+            public Object getVarType(LogicalVariable var) throws AlgebricksException {
+                return null;
+            }
+
+            @Override
+            public Object getType(ILogicalExpression expr) throws AlgebricksException {
+                return ctx.getExpressionTypeComputer().getType(expr, ctx.getMetadataProvider(), this);
+            }
+
+            @Override
+            public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
+                    List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+                return null;
+            }
+
+            @Override
+            public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2)
+                    throws AlgebricksException {
+                return false;
+            }
+        };
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
new file mode 100644
index 0000000..a3c2650
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ExchangeOperator extends AbstractLogicalOperator {
+
+    public ExchangeOperator() {
+        super();
+        setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitExchangeOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+        // do nothing
+        return false;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.EXCHANGE;
+    }
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        AbstractLogicalOperator cld = (AbstractLogicalOperator) inputs.get(0).getValue();
+        OperatorPropertiesUtil.computeSchemaRecIfNull(cld);
+        List<LogicalVariable> inputSchema = inputs.get(0).getValue().getSchema();
+        schema = new ArrayList<LogicalVariable>(inputSchema);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return true;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}

[26/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
new file mode 100644
index 0000000..3abdec9
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
@@ -0,0 +1,281 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.join.OptimizedHybridHashJoinOperatorDescriptor;
+
+public class HybridHashJoinPOperator extends AbstractHashJoinPOperator {
+
+    private final int memSizeInFrames;
+    private final int maxInputBuildSizeInFrames;
+    private final int aveRecordsPerFrame;
+    private final double fudgeFactor;
+
+    private static final Logger LOGGER = Logger.getLogger(HybridHashJoinPOperator.class.getName());
+
+    public HybridHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
+            List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities,
+            int memSizeInFrames, int maxInputSizeInFrames, int aveRecordsPerFrame, double fudgeFactor) {
+        super(kind, partitioningType, sideLeftOfEqualities, sideRightOfEqualities);
+        this.memSizeInFrames = memSizeInFrames;
+        this.maxInputBuildSizeInFrames = maxInputSizeInFrames;
+        this.aveRecordsPerFrame = aveRecordsPerFrame;
+        this.fudgeFactor = fudgeFactor;
+
+        LOGGER.fine("HybridHashJoinPOperator constructed with: JoinKind=" + kind + ", JoinPartitioningType="
+                + partitioningType + ", List<LogicalVariable>=" + sideLeftOfEqualities + ", List<LogicalVariable>="
+                + sideRightOfEqualities + ", int memSizeInFrames=" + memSizeInFrames + ", int maxInputSize0InFrames="
+                + maxInputSizeInFrames + ", int aveRecordsPerFrame=" + aveRecordsPerFrame + ", double fudgeFactor="
+                + fudgeFactor + ".");
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.HYBRID_HASH_JOIN;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    public double getFudgeFactor() {
+        return fudgeFactor;
+    }
+
+    public int getMemSizeInFrames() {
+        return memSizeInFrames;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " " + keysLeftBranch + keysRightBranch;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
+        int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
+                keysLeftBranch, env, context);
+        IBinaryHashFunctionFamily[] hashFunFamilies = JobGenHelper.variablesToBinaryHashFunctionFamilies(
+                keysLeftBranch, env, context);
+        IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
+        int i = 0;
+        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+        for (LogicalVariable v : keysLeftBranch) {
+            Object t = env.getVarType(v);
+            comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
+        }
+
+        IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider = context
+                .getPredicateEvaluatorFactoryProvider();
+        IPredicateEvaluatorFactory predEvaluatorFactory = (predEvaluatorFactoryProvider == null ? null
+                : predEvaluatorFactoryProvider.getPredicateEvaluatorFactory(keysLeft, keysRight));
+
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
+                propagatedSchema, context);
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        IOperatorDescriptor opDesc = null;
+
+        boolean optimizedHashJoin = true;
+        for (IBinaryHashFunctionFamily family : hashFunFamilies) {
+            if (family == null) {
+                optimizedHashJoin = false;
+                break;
+            }
+        }
+
+        if (!optimizedHashJoin) {
+            try {
+                switch (kind) {
+                    case INNER: {
+                        opDesc = new HybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
+                                maxInputBuildSizeInFrames, aveRecordsPerFrame, getFudgeFactor(), keysLeft, keysRight,
+                                hashFunFactories, comparatorFactories, recDescriptor, predEvaluatorFactory);
+                        break;
+                    }
+                    case LEFT_OUTER: {
+                        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
+                        for (int j = 0; j < nullWriterFactories.length; j++) {
+                            nullWriterFactories[j] = context.getNullWriterFactory();
+                        }
+                        opDesc = new HybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
+                                maxInputBuildSizeInFrames, aveRecordsPerFrame, getFudgeFactor(), keysLeft, keysRight,
+                                hashFunFactories, comparatorFactories, recDescriptor, predEvaluatorFactory, true,
+                                nullWriterFactories);
+                        break;
+                    }
+                    default: {
+                        throw new NotImplementedException();
+                    }
+                }
+            } catch (HyracksDataException e) {
+                throw new AlgebricksException(e);
+            }
+        } else {
+            try {
+                switch (kind) {
+                    case INNER: {
+                        opDesc = new OptimizedHybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
+                                maxInputBuildSizeInFrames, getFudgeFactor(), keysLeft, keysRight, hashFunFamilies,
+                                comparatorFactories, recDescriptor, new JoinMultiComparatorFactory(comparatorFactories,
+                                        keysLeft, keysRight), new JoinMultiComparatorFactory(comparatorFactories,
+                                        keysRight, keysLeft), predEvaluatorFactory);
+                        break;
+                    }
+                    case LEFT_OUTER: {
+                        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
+                        for (int j = 0; j < nullWriterFactories.length; j++) {
+                            nullWriterFactories[j] = context.getNullWriterFactory();
+                        }
+                        opDesc = new OptimizedHybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
+                                maxInputBuildSizeInFrames, getFudgeFactor(), keysLeft, keysRight, hashFunFamilies,
+                                comparatorFactories, recDescriptor, new JoinMultiComparatorFactory(comparatorFactories,
+                                        keysLeft, keysRight), new JoinMultiComparatorFactory(comparatorFactories,
+                                        keysRight, keysLeft), predEvaluatorFactory, true, nullWriterFactories);
+                        break;
+                    }
+                    default: {
+                        throw new NotImplementedException();
+                    }
+                }
+            } catch (HyracksDataException e) {
+                throw new AlgebricksException(e);
+            }
+        }
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src1, 0, op, 0);
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
+        builder.contributeGraphEdge(src2, 0, op, 1);
+    }
+
+    @Override
+    protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        return new LinkedList<ILocalStructuralProperty>();
+    }
+
+}
+
+/**
+ * {@ ITuplePairComparatorFactory} implementation for optimized hybrid hash join.
+ */
+class JoinMultiComparatorFactory implements ITuplePairComparatorFactory {
+    private static final long serialVersionUID = 1L;
+
+    private final IBinaryComparatorFactory[] binaryComparatorFactories;
+    private final int[] keysLeft;
+    private final int[] keysRight;
+
+    public JoinMultiComparatorFactory(IBinaryComparatorFactory[] binaryComparatorFactory, int[] keysLeft,
+            int[] keysRight) {
+        this.binaryComparatorFactories = binaryComparatorFactory;
+        this.keysLeft = keysLeft;
+        this.keysRight = keysRight;
+    }
+
+    @Override
+    public ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
+        IBinaryComparator[] binaryComparators = new IBinaryComparator[binaryComparatorFactories.length];
+        for (int i = 0; i < binaryComparators.length; i++) {
+            binaryComparators[i] = binaryComparatorFactories[i].createBinaryComparator();
+        }
+        return new JoinMultiComparator(binaryComparators, keysLeft, keysRight);
+    }
+}
+
+/**
+ * {@ ITuplePairComparator} implementation for optimized hybrid hash join.
+ * The comparator applies multiple binary comparators, one for each key pairs
+ */
+class JoinMultiComparator implements ITuplePairComparator {
+    private final IBinaryComparator[] binaryComparators;
+    private final int[] keysLeft;
+    private final int[] keysRight;
+
+    public JoinMultiComparator(IBinaryComparator[] bComparator, int[] keysLeft, int[] keysRight) {
+        this.binaryComparators = bComparator;
+        this.keysLeft = keysLeft;
+        this.keysRight = keysRight;
+    }
+
+    @Override
+    public int compare(IFrameTupleAccessor accessor0, int tIndex0, IFrameTupleAccessor accessor1, int tIndex1)
+            throws HyracksDataException {
+        int tStart0 = accessor0.getTupleStartOffset(tIndex0);
+        int fStartOffset0 = accessor0.getFieldSlotsLength() + tStart0;
+
+        int tStart1 = accessor1.getTupleStartOffset(tIndex1);
+        int fStartOffset1 = accessor1.getFieldSlotsLength() + tStart1;
+
+        for (int i = 0; i < binaryComparators.length; i++) {
+            int fStart0 = accessor0.getFieldStartOffset(tIndex0, keysLeft[i]);
+            int fEnd0 = accessor0.getFieldEndOffset(tIndex0, keysLeft[i]);
+            int fLen0 = fEnd0 - fStart0;
+
+            int fStart1 = accessor1.getFieldStartOffset(tIndex1, keysRight[i]);
+            int fEnd1 = accessor1.getFieldEndOffset(tIndex1, keysRight[i]);
+            int fLen1 = fEnd1 - fStart1;
+
+            int c = binaryComparators[i].compare(accessor0.getBuffer().array(), fStart0 + fStartOffset0, fLen0,
+                    accessor1.getBuffer().array(), fStart1 + fStartOffset1, fLen1);
+            if (c != 0) {
+                return c;
+            }
+        }
+        return 0;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
new file mode 100644
index 0000000..55616f9
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
+
+public class InMemoryHashJoinPOperator extends AbstractHashJoinPOperator {
+
+    private final int tableSize;
+
+    /**
+     * builds on the first operator and probes on the second.
+     */
+
+    public InMemoryHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
+            List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities, int tableSize) {
+        super(kind, partitioningType, sideLeftOfEqualities, sideRightOfEqualities);
+        this.tableSize = tableSize;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.IN_MEMORY_HASH_JOIN;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " " + keysLeftBranch + keysRightBranch;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
+        int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
+                keysLeftBranch, env, context);
+        IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
+        int i = 0;
+        IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+        for (LogicalVariable v : keysLeftBranch) {
+            Object t = env.getVarType(v);
+            comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, true);
+        }
+        
+        IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider = context.getPredicateEvaluatorFactoryProvider();
+        IPredicateEvaluatorFactory predEvaluatorFactory = ( predEvaluatorFactoryProvider == null ? null : predEvaluatorFactoryProvider.getPredicateEvaluatorFactory(keysLeft, keysRight));
+        
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
+                propagatedSchema, context);
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        IOperatorDescriptor opDesc = null;
+
+        switch (kind) {
+            case INNER: {
+                opDesc = new InMemoryHashJoinOperatorDescriptor(spec, keysLeft, keysRight, hashFunFactories,
+                        comparatorFactories, recDescriptor, tableSize, predEvaluatorFactory);
+                break;
+            }
+            case LEFT_OUTER: {
+                INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
+                for (int j = 0; j < nullWriterFactories.length; j++) {
+                    nullWriterFactories[j] = context.getNullWriterFactory();
+                }
+                opDesc = new InMemoryHashJoinOperatorDescriptor(spec, keysLeft, keysRight, hashFunFactories,
+                        comparatorFactories, predEvaluatorFactory, recDescriptor, true, nullWriterFactories, tableSize);
+                break;
+            }
+            default: {
+                throw new NotImplementedException();
+            }
+        }
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src1, 0, op, 0);
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
+        builder.contributeGraphEdge(src2, 0, op, 1);
+    }
+
+    @Override
+    protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
+        List<ILocalStructuralProperty> lp0 = pv0.getLocalProperties();
+        if (lp0 != null) {
+            // maintains the local properties on the probe side
+            return new LinkedList<ILocalStructuralProperty>(lp0);
+        }
+        return new LinkedList<ILocalStructuralProperty>();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
new file mode 100644
index 0000000..4a7d875
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.sort.InMemorySortRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class InMemoryStableSortPOperator extends AbstractStableSortPOperator {
+
+    public InMemoryStableSortPOperator() {
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.IN_MEMORY_STABLE_SORT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        int n = sortColumns.length;
+        int[] sortFields = new int[n];
+        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+        int i = 0;
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        INormalizedKeyComputerFactory nkcf = null;
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        for (OrderColumn oc : sortColumns) {
+            LogicalVariable var = oc.getColumn();
+            sortFields[i] = opSchema.findVariable(var);
+            Object type = env.getVarType(var);
+            OrderKind order = oc.getOrder();
+            if (i == 0 && nkcfProvider != null && type != null) {
+                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
+            }
+
+            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
+            i++;
+        }
+
+        IPushRuntimeFactory runtime = new InMemorySortRuntimeFactory(sortFields, nkcf, comps, null);
+        builder.contributeMicroOperator(op, runtime, recDescriptor);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
new file mode 100644
index 0000000..db00b9e
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexBulkloadPOperator.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class IndexBulkloadPOperator extends AbstractPhysicalOperator {
+
+    private final List<LogicalVariable> primaryKeys;
+    private final List<LogicalVariable> secondaryKeys;
+    private final List<LogicalVariable> additionalFilteringKeys;
+    private final Mutable<ILogicalExpression> filterExpr;
+    private final IDataSourceIndex<?, ?> dataSourceIndex;
+
+    public IndexBulkloadPOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            List<LogicalVariable> additionalFilteringKeys, Mutable<ILogicalExpression> filterExpr,
+            IDataSourceIndex<?, ?> dataSourceIndex) {
+        this.primaryKeys = primaryKeys;
+        this.secondaryKeys = secondaryKeys;
+        this.additionalFilteringKeys = additionalFilteringKeys;
+        this.filterExpr = filterExpr;
+        this.dataSourceIndex = dataSourceIndex;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.INDEX_BULKLOAD;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<LogicalVariable> scanVariables = new ArrayList<>();
+        scanVariables.addAll(primaryKeys);
+        scanVariables.add(new LogicalVariable(-1));
+        IPhysicalPropertiesVector physicalProps = dataSourceIndex.getDataSource().getPropertiesProvider()
+                .computePropertiesVector(scanVariables);
+        List<ILocalStructuralProperty> localProperties = new ArrayList<>();
+        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+        // Data needs to be sorted based on the [token, number of token, PK]
+        // OR [token, PK] if the index is not partitioned
+        for (LogicalVariable skVar : secondaryKeys) {
+            orderColumns.add(new OrderColumn(skVar, OrderKind.ASC));
+        }
+        for (LogicalVariable pkVar : primaryKeys) {
+            orderColumns.add(new OrderColumn(pkVar, OrderKind.ASC));
+        }
+        localProperties.add(new LocalOrderProperty(orderColumns));
+        StructuralPropertiesVector spv = new StructuralPropertiesVector(physicalProps.getPartitioningProperty(),
+                localProperties);
+        return new PhysicalRequirements(new IPhysicalPropertiesVector[] { spv },
+                IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        IndexInsertDeleteOperator indexInsertDeleteOp = (IndexInsertDeleteOperator) op;
+        assert indexInsertDeleteOp.getOperation() == Kind.INSERT;
+        assert indexInsertDeleteOp.isBulkload();
+
+        IMetadataProvider mp = context.getMetadataProvider();
+        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
+        JobSpecification spec = builder.getJobSpec();
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getIndexInsertRuntime(
+                dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys,
+                additionalFilteringKeys, null, inputDesc, context, spec, true);
+        builder.contributeHyracksOperator(indexInsertDeleteOp, runtimeAndConstraints.first);
+        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+        ILogicalOperator src = indexInsertDeleteOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, indexInsertDeleteOp, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
new file mode 100644
index 0000000..4bc4c06
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class IndexInsertDeletePOperator extends AbstractPhysicalOperator {
+
+    private final List<LogicalVariable> primaryKeys;
+    private final List<LogicalVariable> secondaryKeys;
+    private final ILogicalExpression filterExpr;
+    private final IDataSourceIndex<?, ?> dataSourceIndex;
+    private final List<LogicalVariable> additionalFilteringKeys;
+
+    public IndexInsertDeletePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            List<LogicalVariable> additionalFilteringKeys, Mutable<ILogicalExpression> filterExpr,
+            IDataSourceIndex<?, ?> dataSourceIndex) {
+        this.primaryKeys = primaryKeys;
+        this.secondaryKeys = secondaryKeys;
+        if (filterExpr != null) {
+            this.filterExpr = filterExpr.getValue();
+        } else {
+            this.filterExpr = null;
+        }
+        this.dataSourceIndex = dataSourceIndex;
+        this.additionalFilteringKeys = additionalFilteringKeys;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.INDEX_INSERT_DELETE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
+        scanVariables.addAll(primaryKeys);
+        scanVariables.add(new LogicalVariable(-1));
+        IPhysicalPropertiesVector r = dataSourceIndex.getDataSource().getPropertiesProvider()
+                .computePropertiesVector(scanVariables);
+        r.getLocalProperties().clear();
+        IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
+        requirements[0] = r;
+        return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        IndexInsertDeleteOperator insertDeleteOp = (IndexInsertDeleteOperator) op;
+        IMetadataProvider mp = context.getMetadataProvider();
+
+        JobSpecification spec = builder.getJobSpec();
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
+
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
+        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(insertDeleteOp);
+        if (insertDeleteOp.getOperation() == Kind.INSERT) {
+            runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
+                    primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec, false);
+        } else {
+            runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv,
+                    primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec);
+        }
+        builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
+        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
new file mode 100644
index 0000000..993e8d7
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+@SuppressWarnings("rawtypes")
+public class InsertDeletePOperator extends AbstractPhysicalOperator {
+
+    private LogicalVariable payload;
+    private List<LogicalVariable> keys;
+    private IDataSource<?> dataSource;
+    private final List<LogicalVariable> additionalFilteringKeys;
+
+    public InsertDeletePOperator(LogicalVariable payload, List<LogicalVariable> keys,
+            List<LogicalVariable> additionalFilteringKeys, IDataSource dataSource) {
+        this.payload = payload;
+        this.keys = keys;
+        this.dataSource = dataSource;
+        this.additionalFilteringKeys = additionalFilteringKeys;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.INSERT_DELETE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
+        scanVariables.addAll(keys);
+        scanVariables.add(new LogicalVariable(-1));
+        IPhysicalPropertiesVector r = dataSource.getPropertiesProvider().computePropertiesVector(scanVariables);
+        r.getLocalProperties().clear();
+        IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
+        requirements[0] = r;
+        return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
+        IMetadataProvider mp = context.getMetadataProvider();
+        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
+        JobSpecification spec = builder.getJobSpec();
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
+
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
+        if (insertDeleteOp.getOperation() == Kind.INSERT) {
+            runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
+                    additionalFilteringKeys, inputDesc, context, spec, false);
+        } else {
+            runtimeAndConstraints = mp.getDeleteRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
+                    additionalFilteringKeys, inputDesc, context, spec);
+        }
+
+        builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
+        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java
new file mode 100644
index 0000000..27f4b25
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.misc.MaterializingOperatorDescriptor;
+
+public class MaterializePOperator extends AbstractPhysicalOperator {
+
+    private final boolean isSingleActivity;
+
+    public MaterializePOperator(boolean isSingleActivity) {
+        this.isSingleActivity = isSingleActivity;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.MATERIALIZE;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
+                propagatedSchema, context);
+        MaterializingOperatorDescriptor materializationOpDesc = new MaterializingOperatorDescriptor(
+                builder.getJobSpec(), recDescriptor, isSingleActivity);
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, materializationOpDesc);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[] { 0 };
+        int[] outputDependencyLabels;
+        if (isSingleActivity) {
+            outputDependencyLabels = new int[] { 0 };
+        } else {
+            outputDependencyLabels = new int[] { 1 };
+        }
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
new file mode 100644
index 0000000..546a77b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class MicroPreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
+
+    public MicroPreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
+        super(columnList);
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
+        GroupByOperator gby = (GroupByOperator) op;
+        int numFds = gby.getDecorList().size();
+        int fdColumns[] = new int[numFds];
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        int j = 0;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new AlgebricksException("pre-sorted group-by expects variable references.");
+            }
+            VariableReferenceExpression v = (VariableReferenceExpression) expr;
+            LogicalVariable decor = v.getVariableReference();
+            fdColumns[j++] = inputSchemas[0].findVariable(decor);
+        }
+        // compile subplans and set the gby op. schema accordingly
+        AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
+        IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys,
+                fdColumns);
+
+        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
+                columnList, env, context);
+        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()),
+                inputSchemas[0], context);
+        MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys,
+                comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null);
+        builder.contributeMicroOperator(gby, runtime, recordDescriptor);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
new file mode 100644
index 0000000..a2e697d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
@@ -0,0 +1,288 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
+
+/**
+ * Left input is broadcast and preserves its local properties.
+ * Right input can be partitioned in any way.
+ */
+public class NLJoinPOperator extends AbstractJoinPOperator {
+
+    private final int memSize;
+
+    public NLJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType, int memSize) {
+        super(kind, partitioningType);
+        this.memSize = memSize;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.NESTED_LOOP;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context) {
+        if (partitioningType != JoinPartitioningType.BROADCAST) {
+            throw new NotImplementedException(partitioningType + " nested loop joins are not implemented.");
+        }
+
+        IPartitioningProperty pp;
+
+        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+
+        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
+            IPhysicalPropertiesVector pv1 = op2.getPhysicalOperator().getDeliveredProperties();
+            if (pv1 == null) {
+                pp = null;
+            } else {
+                pp = pv1.getPartitioningProperty();
+            }
+        } else {
+            pp = IPartitioningProperty.UNPARTITIONED;
+        }
+
+        List<ILocalStructuralProperty> localProps = new LinkedList<ILocalStructuralProperty>();
+        this.deliveredProperties = new StructuralPropertiesVector(pp, localProps);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        if (partitioningType != JoinPartitioningType.BROADCAST) {
+            throw new NotImplementedException(partitioningType + " nested loop joins are not implemented.");
+        }
+
+        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
+        pv[0] = new StructuralPropertiesVector(new BroadcastPartitioningProperty(null), null);
+        pv[1] = new StructuralPropertiesVector(null, null);
+        return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
+                propagatedSchema, context);
+        IOperatorSchema[] conditionInputSchemas = new IOperatorSchema[1];
+        conditionInputSchemas[0] = propagatedSchema;
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(join.getCondition().getValue(),
+                context.getTypeEnvironment(op), conditionInputSchemas, context);
+        ITuplePairComparatorFactory comparatorFactory = new TuplePairEvaluatorFactory(cond,
+                context.getBinaryBooleanInspectorFactory());
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        IOperatorDescriptor opDesc = null;
+
+        switch (kind) {
+            case INNER: {
+                opDesc = new NestedLoopJoinOperatorDescriptor(spec, comparatorFactory, recDescriptor, memSize, false,
+                        null);
+                break;
+            }
+            case LEFT_OUTER: {
+                INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
+                for (int j = 0; j < nullWriterFactories.length; j++) {
+                    nullWriterFactories[j] = context.getNullWriterFactory();
+                }
+                opDesc = new NestedLoopJoinOperatorDescriptor(spec, comparatorFactory, recDescriptor, memSize, true,
+                        nullWriterFactories);
+                break;
+            }
+            default: {
+                throw new NotImplementedException();
+            }
+        }
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src1, 0, op, 0);
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
+        builder.contributeGraphEdge(src2, 0, op, 1);
+    }
+
+    public static class TuplePairEvaluatorFactory implements ITuplePairComparatorFactory {
+
+        private static final long serialVersionUID = 1L;
+        private final IScalarEvaluatorFactory cond;
+        private final IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
+
+        public TuplePairEvaluatorFactory(IScalarEvaluatorFactory cond,
+                IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
+            this.cond = cond;
+            this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
+        }
+
+        @Override
+        public synchronized ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
+            return new TuplePairEvaluator(ctx, cond, binaryBooleanInspectorFactory.createBinaryBooleanInspector(ctx));
+        }
+    }
+
+    public static class TuplePairEvaluator implements ITuplePairComparator {
+        private final IHyracksTaskContext ctx;
+        private IScalarEvaluator condEvaluator;
+        private final IScalarEvaluatorFactory condFactory;
+        private final IPointable p;
+        private final CompositeFrameTupleReference compositeTupleRef;
+        private final FrameTupleReference leftRef;
+        private final FrameTupleReference rightRef;
+        private final IBinaryBooleanInspector binaryBooleanInspector;
+
+        public TuplePairEvaluator(IHyracksTaskContext ctx, IScalarEvaluatorFactory condFactory,
+                IBinaryBooleanInspector binaryBooleanInspector) {
+            this.ctx = ctx;
+            this.condFactory = condFactory;
+            this.binaryBooleanInspector = binaryBooleanInspector;
+            this.leftRef = new FrameTupleReference();
+            this.p = VoidPointable.FACTORY.createPointable();
+            this.rightRef = new FrameTupleReference();
+            this.compositeTupleRef = new CompositeFrameTupleReference(leftRef, rightRef);
+        }
+
+        @Override
+        public int compare(IFrameTupleAccessor outerAccessor, int outerIndex, IFrameTupleAccessor innerAccessor,
+                int innerIndex) throws HyracksDataException {
+            if (condEvaluator == null) {
+                try {
+                    this.condEvaluator = condFactory.createScalarEvaluator(ctx);
+                } catch (AlgebricksException ae) {
+                    throw new HyracksDataException(ae);
+                }
+            }
+            compositeTupleRef.reset(outerAccessor, outerIndex, innerAccessor, innerIndex);
+            try {
+                condEvaluator.evaluate(compositeTupleRef, p);
+            } catch (AlgebricksException ae) {
+                throw new HyracksDataException(ae);
+            }
+            boolean result = binaryBooleanInspector
+                    .getBooleanValue(p.getByteArray(), p.getStartOffset(), p.getLength());
+            if (result)
+                return 0;
+            else
+                return 1;
+        }
+    }
+
+    public static class CompositeFrameTupleReference implements IFrameTupleReference {
+
+        private final FrameTupleReference refLeft;
+        private final FrameTupleReference refRight;
+
+        public CompositeFrameTupleReference(FrameTupleReference refLeft, FrameTupleReference refRight) {
+            this.refLeft = refLeft;
+            this.refRight = refRight;
+        }
+
+        public void reset(IFrameTupleAccessor outerAccessor, int outerIndex, IFrameTupleAccessor innerAccessor,
+                int innerIndex) {
+            refLeft.reset(outerAccessor, outerIndex);
+            refRight.reset(innerAccessor, innerIndex);
+        }
+
+        @Override
+        public int getFieldCount() {
+            return refLeft.getFieldCount() + refRight.getFieldCount();
+        }
+
+        @Override
+        public byte[] getFieldData(int fIdx) {
+            int leftFieldCount = refLeft.getFieldCount();
+            if (fIdx < leftFieldCount)
+                return refLeft.getFieldData(fIdx);
+            else
+                return refRight.getFieldData(fIdx - leftFieldCount);
+        }
+
+        @Override
+        public int getFieldStart(int fIdx) {
+            int leftFieldCount = refLeft.getFieldCount();
+            if (fIdx < leftFieldCount)
+                return refLeft.getFieldStart(fIdx);
+            else
+                return refRight.getFieldStart(fIdx - leftFieldCount);
+        }
+
+        @Override
+        public int getFieldLength(int fIdx) {
+            int leftFieldCount = refLeft.getFieldCount();
+            if (fIdx < leftFieldCount)
+                return refLeft.getFieldLength(fIdx);
+            else
+                return refRight.getFieldLength(fIdx - leftFieldCount);
+        }
+
+        @Override
+        public IFrameTupleAccessor getFrameTupleAccessor() {
+            throw new NotImplementedException();
+        }
+
+        @Override
+        public int getTupleIndex() {
+            throw new NotImplementedException();
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
new file mode 100644
index 0000000..4fdfa32
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class NestedTupleSourcePOperator extends AbstractPhysicalOperator {
+
+    public NestedTupleSourcePOperator() {
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.NESTED_TUPLE_SOURCE;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        Mutable<ILogicalOperator> dataSource = ((NestedTupleSourceOperator) op).getDataSourceReference();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) dataSource.getValue().getInputs().get(0).getValue();
+        IPhysicalPropertiesVector inheritedProps = op2.getDeliveredPhysicalProperties();
+        AbstractLogicalOperator parent = (AbstractLogicalOperator) dataSource.getValue();
+        if (parent.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            // The following part computes the data property regarding to each particular group.
+            // TODO(buyingyi): we need to add the original data property as well. But currently
+            // there are places assuming there is only one LocalOrderProperty and one
+            // LocalGroupingProperty delivered by an operator.
+            GroupByOperator gby = (GroupByOperator) parent;
+            List<ILocalStructuralProperty> originalLocalProperties = inheritedProps.getLocalProperties();
+            List<ILocalStructuralProperty> newLocalProperties = null;
+            if (originalLocalProperties != null) {
+                newLocalProperties = new ArrayList<ILocalStructuralProperty>();
+                for (ILocalStructuralProperty lsp : inheritedProps.getLocalProperties()) {
+                    ILocalStructuralProperty newLsp = lsp.regardToGroup(gby.getGbyVarList());
+                    if (newLsp != null) {
+                        newLocalProperties.add(newLsp);
+                    }
+                }
+            }
+            deliveredProperties = new StructuralPropertiesVector(inheritedProps.getPartitioningProperty(),
+                    newLocalProperties);
+        } else {
+            deliveredProperties = inheritedProps.clone();
+        }
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return null;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        propagatedSchema.addAllVariables(outerPlanSchema);
+        NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory();
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
+                context);
+        builder.contributeMicroOperator(op, runtime, recDesc);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
new file mode 100644
index 0000000..456744e
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+
+public class OneToOneExchangePOperator extends AbstractExchangePOperator {
+
+    public OneToOneExchangePOperator() {
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
+        IConnectorDescriptor conn = new OneToOneConnectorDescriptor(spec);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.SAME_COUNT);
+    }
+
+}

[30/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
new file mode 100644
index 0000000..1d7f134
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -0,0 +1,763 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
+        ctx.putFDList(op, new ArrayList<FunctionalDependency>());
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+        ctx.putEquivalenceClassMap(op, eqClasses);
+
+        // Propagates equivalence classes that from expressions.
+        // Note that an equivalence class can also contain expression members.
+        propagateEquivalenceFromExpressionsToVariables(eqClasses, op.getExpressions(), op.getVariables());
+
+        // Generates FDs.
+        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op, used);
+        List<FunctionalDependency> fds1 = getOrComputeFDs(inp1, ctx);
+        List<FunctionalDependency> eFds = new ArrayList<FunctionalDependency>(fds1.size());
+        for (FunctionalDependency fd : fds1) {
+            if (fd.getTail().containsAll(used)) {
+                List<LogicalVariable> hd = new ArrayList<LogicalVariable>(fd.getHead());
+                List<LogicalVariable> tl = new ArrayList<LogicalVariable>(fd.getTail());
+                tl.addAll(op.getVariables());
+                FunctionalDependency fd2 = new FunctionalDependency(hd, tl);
+                eFds.add(fd2);
+            } else {
+                eFds.add(fd);
+            }
+        }
+        ctx.putFDList(op, eFds);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
+        Map<LogicalVariable, EquivalenceClass> propagatedEqClasses = getOrComputeEqClasses(inp1, ctx);
+        eqClasses.putAll(propagatedEqClasses);
+        ctx.putEquivalenceClassMap(op, eqClasses);
+        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
+        ctx.putFDList(op, fds);
+        op.getDataSource().computeFDs(op.getVariables(), fds);
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
+        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+        ctx.putFDList(op, functionalDependencies);
+        for (FunctionalDependency inherited : getOrComputeFDs(op0, ctx)) {
+            boolean isCoveredByDistinctByVars = true;
+            for (LogicalVariable v : inherited.getHead()) {
+                if (!op.isDistinctByVar(v)) {
+                    isCoveredByDistinctByVars = false;
+                }
+            }
+            if (isCoveredByDistinctByVars) {
+                List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
+                for (LogicalVariable v2 : inherited.getTail()) {
+                    if (op.isDistinctByVar(v2)) {
+                        newTail.add(v2);
+                    }
+                }
+                if (!newTail.isEmpty()) {
+                    List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead());
+                    FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
+                    functionalDependencies.add(newFd);
+                }
+            }
+        }
+        Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
+        List<Mutable<ILogicalExpression>> expressions = op.getExpressions();
+        for (Mutable<ILogicalExpression> pRef : expressions) {
+            ILogicalExpression p = pRef.getValue();
+            if (p.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) p;
+                gbySet.add(v.getVariableReference());
+            }
+        }
+        LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
+
+        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = getOrComputeEqClasses(op0, ctx);
+        ctx.putEquivalenceClassMap(op, equivalenceClasses);
+
+        lgp.normalizeGroupingColumns(equivalenceClasses, functionalDependencies);
+        Set<LogicalVariable> normSet = lgp.getColumnSet();
+        List<Mutable<ILogicalExpression>> newDistinctByList = new ArrayList<Mutable<ILogicalExpression>>();
+        for (Mutable<ILogicalExpression> p2Ref : expressions) {
+            ILogicalExpression p2 = p2Ref.getValue();
+            if (p2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression var2 = (VariableReferenceExpression) p2;
+                LogicalVariable v2 = var2.getVariableReference();
+                if (normSet.contains(v2)) {
+                    newDistinctByList.add(p2Ref);
+                }
+            } else {
+                newDistinctByList.add(p2Ref);
+            }
+        }
+        expressions.clear();
+        expressions.addAll(newDistinctByList);
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
+        ctx.putFDList(op, new ArrayList<FunctionalDependency>());
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+        ctx.putEquivalenceClassMap(op, equivalenceClasses);
+        ctx.putFDList(op, functionalDependencies);
+
+        List<FunctionalDependency> inheritedFDs = new ArrayList<FunctionalDependency>();
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator op2 = r.getValue();
+                equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
+                inheritedFDs.addAll(getOrComputeFDs(op2, ctx));
+            }
+        }
+
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
+        inheritedFDs.addAll(getOrComputeFDs(op0, ctx));
+        Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx);
+        for (FunctionalDependency inherited : inheritedFDs) {
+            boolean isCoveredByGbyOrDecorVars = true;
+            List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead().size());
+            for (LogicalVariable v : inherited.getHead()) {
+                LogicalVariable vnew = getNewGbyVar(op, v);
+                if (vnew == null) {
+                    vnew = getNewDecorVar(op, v);
+                    if (vnew == null) {
+                        isCoveredByGbyOrDecorVars = false;
+                    }
+                    break;
+                }
+                newHead.add(vnew);
+            }
+
+            if (isCoveredByGbyOrDecorVars) {
+                List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
+                for (LogicalVariable v2 : inherited.getTail()) {
+                    LogicalVariable v3 = getNewGbyVar(op, v2);
+                    if (v3 != null) {
+                        newTail.add(v3);
+                    }
+                }
+                if (!newTail.isEmpty()) {
+                    FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
+                    functionalDependencies.add(newFd);
+                }
+            }
+        }
+
+        List<LogicalVariable> premiseGby = new LinkedList<LogicalVariable>();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            premiseGby.add(p.first);
+        }
+
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList();
+
+        LinkedList<LogicalVariable> conclDecor = new LinkedList<LogicalVariable>();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+            conclDecor.add(GroupByOperator.getDecorVariable(p));
+        }
+        if (!conclDecor.isEmpty()) {
+            functionalDependencies.add(new FunctionalDependency(premiseGby, conclDecor));
+        }
+
+        Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) expr;
+                gbySet.add(v.getVariableReference());
+            }
+        }
+        LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
+        lgp.normalizeGroupingColumns(inheritedEcs, inheritedFDs);
+        Set<LogicalVariable> normSet = lgp.getColumnSet();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newGbyList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+        boolean changed = false;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+                LogicalVariable v2 = varRef.getVariableReference();
+                EquivalenceClass ec2 = inheritedEcs.get(v2);
+                LogicalVariable v3;
+                if (ec2 != null && !ec2.representativeIsConst()) {
+                    v3 = ec2.getVariableRepresentative();
+                } else {
+                    v3 = v2;
+                }
+                if (normSet.contains(v3)) {
+                    newGbyList.add(p);
+                } else {
+                    changed = true;
+                    decorList.add(p);
+                }
+            } else {
+                newGbyList.add(p);
+            }
+        }
+        if (changed) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Group-by list changed from "
+                    + GroupByOperator.veListToString(gByList) + " to " + GroupByOperator.veListToString(newGbyList)
+                    + ".\n");
+        }
+        gByList.clear();
+        gByList.addAll(newGbyList);
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+        ctx.putEquivalenceClassMap(op, equivalenceClasses);
+        ctx.putFDList(op, functionalDependencies);
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
+        ILogicalOperator op1 = op.getInputs().get(1).getValue();
+        functionalDependencies.addAll(getOrComputeFDs(op0, ctx));
+        functionalDependencies.addAll(getOrComputeFDs(op1, ctx));
+        equivalenceClasses.putAll(getOrComputeEqClasses(op0, ctx));
+        equivalenceClasses.putAll(getOrComputeEqClasses(op1, ctx));
+        ILogicalExpression expr = op.getCondition().getValue();
+        expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+        ctx.putEquivalenceClassMap(op, equivalenceClasses);
+        ctx.putFDList(op, functionalDependencies);
+        ILogicalOperator opLeft = op.getInputs().get(0).getValue();
+        ILogicalOperator opRight = op.getInputs().get(1).getValue();
+        functionalDependencies.addAll(getOrComputeFDs(opLeft, ctx));
+        functionalDependencies.addAll(getOrComputeFDs(opRight, ctx));
+        equivalenceClasses.putAll(getOrComputeEqClasses(opLeft, ctx));
+        equivalenceClasses.putAll(getOrComputeEqClasses(opRight, ctx));
+
+        Collection<LogicalVariable> leftSideVars;
+        if (opLeft.getSchema() == null) {
+            leftSideVars = new LinkedList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(opLeft, leftSideVars);
+            // actually, not all produced vars. are visible (due to projection)
+            // so using cached schema is better and faster
+        } else {
+            leftSideVars = opLeft.getSchema();
+        }
+        ILogicalExpression expr = op.getCondition().getValue();
+        expr.getConstraintsForOuterJoin(functionalDependencies, leftSideVars);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getDataSourceReference().getValue();
+        ILogicalOperator inp1 = op1.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+        ctx.putEquivalenceClassMap(op, eqClasses);
+        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
+        if (op1.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            GroupByOperator gby = (GroupByOperator) op1;
+            LinkedList<LogicalVariable> tail = new LinkedList<LogicalVariable>();
+            for (LogicalVariable v : gby.getGbyVarList()) {
+                tail.add(v);
+                // all values for gby vars. are the same
+            }
+            FunctionalDependency gbyfd = new FunctionalDependency(new LinkedList<LogicalVariable>(), tail);
+            fds.add(gbyfd);
+        }
+        ctx.putFDList(op, fds);
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        throw new NotImplementedException();
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClassesForUsedVars(op, ctx, op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
+        ctx.putFDList(op, new ArrayList<FunctionalDependency>());
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClassesForUsedVars(op, ctx, op.getInputVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        ILogicalOperator childOp = op.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = getOrComputeEqClasses(childOp, ctx);
+        ctx.putEquivalenceClassMap(op, equivalenceClasses);
+
+        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+        ctx.putFDList(op, functionalDependencies);
+        functionalDependencies.addAll(getOrComputeFDs(childOp, ctx));
+        equivalenceClasses.putAll(getOrComputeEqClasses(childOp, ctx));
+        ILogicalExpression expr = op.getCondition().getValue();
+        expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+        ctx.putEquivalenceClassMap(op, equivalenceClasses);
+        ctx.putFDList(op, functionalDependencies);
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator op2 = r.getValue();
+                equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
+                functionalDependencies.addAll(getOrComputeFDs(op2, ctx));
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        setEmptyFDsEqClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        fdsEqClassesForAbstractUnnestOperator(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        fdsEqClassesForAbstractUnnestOperator(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        // propagateFDsAndEquivClasses(op, ctx);
+        setEmptyFDsEqClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        // propagateFDsAndEquivClasses(op, ctx);
+        setEmptyFDsEqClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        // propagateFDsAndEquivClasses(op, ctx);
+        setEmptyFDsEqClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        setEmptyFDsEqClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        setEmptyFDsEqClasses(op, ctx);
+        return null;
+    }
+
+    private void propagateFDsAndEquivClasses(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+        ctx.putEquivalenceClassMap(op, eqClasses);
+        List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
+        ctx.putFDList(op, fds);
+    }
+
+    private Map<LogicalVariable, EquivalenceClass> getOrComputeEqClasses(ILogicalOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        Map<LogicalVariable, EquivalenceClass> eqClasses = ctx.getEquivalenceClassMap(op);
+        if (eqClasses == null) {
+            op.accept(this, ctx);
+            eqClasses = ctx.getEquivalenceClassMap(op);
+        }
+        return eqClasses;
+    }
+
+    private Map<LogicalVariable, EquivalenceClass> getOrCreateEqClasses(ILogicalOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        Map<LogicalVariable, EquivalenceClass> eqClasses = ctx.getEquivalenceClassMap(op);
+        if (eqClasses == null) {
+            eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+            ctx.putEquivalenceClassMap(op, eqClasses);
+        }
+        return eqClasses;
+    }
+
+    private List<FunctionalDependency> getOrComputeFDs(ILogicalOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        List<FunctionalDependency> fds = ctx.getFDList(op);
+        if (fds == null) {
+            op.accept(this, ctx);
+            fds = ctx.getFDList(op);
+        }
+        return fds;
+    }
+
+    /***
+     * Propagated equivalent classes from the child to the current operator, based
+     * on the used variables of the current operator.
+     * 
+     * @param op
+     *            , the current operator
+     * @param ctx
+     *            , the optimization context which keeps track of all equivalent classes.
+     * @param usedVariables
+     *            , used variables.
+     * @throws AlgebricksException
+     */
+    private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx,
+            List<LogicalVariable> usedVariables) throws AlgebricksException {
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
+        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
+        ctx.putFDList(op, fds);
+
+        Map<LogicalVariable, EquivalenceClass> chldClasses = getOrComputeEqClasses(op2, ctx);
+
+        // Propagate equivalent classes that contain the used variables.
+        for (LogicalVariable v : usedVariables) {
+            EquivalenceClass ec = eqClasses.get(v);
+            if (ec == null) {
+                EquivalenceClass oc = chldClasses.get(v);
+                if (oc == null) {
+                    continue;
+                }
+                List<LogicalVariable> m = new LinkedList<LogicalVariable>();
+                for (LogicalVariable v2 : oc.getMembers()) {
+                    if (usedVariables.contains(v2)) {
+                        m.add(v2);
+                    }
+                }
+                EquivalenceClass nc;
+                if (oc.representativeIsConst()) {
+                    nc = new EquivalenceClass(m, oc.getConstRepresentative());
+                } else if (m.contains(oc.getVariableRepresentative())) {
+                    nc = new EquivalenceClass(m, oc.getVariableRepresentative());
+                } else {
+                    nc = new EquivalenceClass(m, v);
+                }
+                for (LogicalVariable v3 : m) {
+                    eqClasses.put(v3, nc);
+                }
+            }
+        }
+
+        // Propagates equivalent classes that contain expressions that use the used variables.
+        // Note that for the case variable $v is not in the used variables but it is
+        // equivalent to field-access($t, i) and $t is a used variable, the equivalent
+        // class should still be propagated (kept).
+        Set<LogicalVariable> usedVarSet = new HashSet<LogicalVariable>(usedVariables);
+        for (Entry<LogicalVariable, EquivalenceClass> entry : chldClasses.entrySet()) {
+            EquivalenceClass ec = entry.getValue();
+            for (ILogicalExpression expr : ec.getExpressionMembers()) {
+                Set<LogicalVariable> exprUsedVars = new HashSet<LogicalVariable>();
+                expr.getUsedVariables(exprUsedVars);
+                exprUsedVars.retainAll(usedVarSet);
+                // Check if the expression member uses a used variable.
+                if (!exprUsedVars.isEmpty()) {
+                    for (LogicalVariable v : ec.getMembers()) {
+                        eqClasses.put(v, ec);
+                        // If variable members contain a used variable, the representative
+                        // variable should be a used variable.
+                        if (usedVarSet.contains(v)) {
+                            ec.setVariableRepresentative(v);
+                        }
+                    }
+                }
+            }
+        }
+
+        List<FunctionalDependency> chldFds = getOrComputeFDs(op2, ctx);
+        for (FunctionalDependency fd : chldFds) {
+            if (!usedVariables.containsAll(fd.getHead())) {
+                continue;
+            }
+            List<LogicalVariable> tl = new LinkedList<LogicalVariable>();
+            for (LogicalVariable v : fd.getTail()) {
+                if (usedVariables.contains(v)) {
+                    tl.add(v);
+                }
+            }
+            if (!tl.isEmpty()) {
+                FunctionalDependency newFd = new FunctionalDependency(fd.getHead(), tl);
+                fds.add(newFd);
+            }
+        }
+    }
+
+    private void fdsEqClassesForAbstractUnnestOperator(AbstractUnnestOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
+        Map<LogicalVariable, EquivalenceClass> propagatedEqClasses = getOrComputeEqClasses(inp1, ctx);
+        /**
+         * The original eq classes of unnest-map are only for produced variables, therefore
+         * eqClasses and propagatedEqClasses do not have overlaps.
+         */
+        eqClasses.putAll(propagatedEqClasses);
+        ctx.putEquivalenceClassMap(op, eqClasses);
+        List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
+        ctx.putFDList(op, fds);
+
+        ILogicalExpression expr = op.getExpressionRef().getValue();
+        if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+            AbstractFunctionCallExpression afe = (AbstractFunctionCallExpression) expr;
+            if (afe.getKind() == FunctionKind.UNNEST && ((UnnestingFunctionCallExpression) afe).returnsUniqueValues()) {
+                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+                VariableUtilities.getLiveVariables(op, vars);
+                ArrayList<LogicalVariable> h = new ArrayList<LogicalVariable>();
+                h.addAll(op.getVariables());
+                FunctionalDependency fd = new FunctionalDependency(h, vars);
+                fds.add(fd);
+            }
+        }
+    }
+
+    public static void setEmptyFDsEqClasses(ILogicalOperator op, IOptimizationContext ctx) {
+        Map<LogicalVariable, EquivalenceClass> eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+        ctx.putEquivalenceClassMap(op, eqClasses);
+        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
+        ctx.putFDList(op, fds);
+    }
+
+    private LogicalVariable getNewGbyVar(GroupByOperator g, LogicalVariable v) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getGroupByList()) {
+            ILogicalExpression e = p.second.getValue();
+            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
+                if (v2 == v) {
+                    return p.first;
+                }
+            }
+        }
+        return null;
+    }
+
+    private LogicalVariable getNewDecorVar(GroupByOperator g, LogicalVariable v) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
+            ILogicalExpression e = p.second.getValue();
+            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
+                if (v2 == v) {
+                    return (p.first != null) ? p.first : v2;
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, IOptimizationContext ctx) throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        propagateFDsAndEquivClasses(op, ctx);
+        return null;
+    }
+
+    /**
+     * Propagate equivalences that carried in expressions to the variables that
+     * they are assigned to.
+     * 
+     * @param eqClasses
+     *            an equivalent class map
+     * @param assignExprs
+     *            expressions on the right-hand-side of assignments
+     * @param assignVars
+     *            variables on the left-hand-side of assignments
+     */
+    private void propagateEquivalenceFromExpressionsToVariables(Map<LogicalVariable, EquivalenceClass> eqClasses,
+            List<Mutable<ILogicalExpression>> assignExprs, List<LogicalVariable> assignVars) {
+        for (int assignVarIndex = 0; assignVarIndex < assignVars.size(); ++assignVarIndex) {
+            LogicalVariable var = assignVars.get(assignVarIndex);
+            ILogicalExpression expr = assignExprs.get(assignVarIndex).getValue();
+            for (Entry<LogicalVariable, EquivalenceClass> entry : eqClasses.entrySet()) {
+                EquivalenceClass eqc = entry.getValue();
+                // If the equivalence class contains the right-hand-side expression,
+                // the left-hand-side variable is added into the equivalence class.
+                if (eqc.contains(expr)) {
+                    eqc.addMember(var);
+                }
+            }
+        }
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
new file mode 100644
index 0000000..b9b111d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -0,0 +1,575 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class IsomorphismOperatorVisitor implements ILogicalOperatorVisitor<Boolean, ILogicalOperator> {
+
+    private final Map<LogicalVariable, LogicalVariable> variableMapping = new HashMap<LogicalVariable, LogicalVariable>();
+
+    public IsomorphismOperatorVisitor() {
+    }
+
+    @Override
+    public Boolean visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.AGGREGATE)
+            return Boolean.FALSE;
+        AggregateOperator aggOpArg = (AggregateOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(
+                getPairList(op.getVariables(), op.getExpressions()),
+                getPairList(aggOpArg.getVariables(), aggOpArg.getExpressions()));
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.RUNNINGAGGREGATE)
+            return Boolean.FALSE;
+        RunningAggregateOperator aggOpArg = (RunningAggregateOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(
+                getPairList(op.getVariables(), op.getExpressions()),
+                getPairList(aggOpArg.getVariables(), aggOpArg.getExpressions()));
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) copyAndSubstituteVar(op, arg);
+        if (aop.getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE)
+            return Boolean.FALSE;
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException {
+        ExtensionOperator aop = (ExtensionOperator) copyAndSubstituteVar(op, arg);
+        if (aop.getOperatorTag() != LogicalOperatorTag.EXTENSION_OPERATOR)
+            return Boolean.FALSE;
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        // require the same physical operator, otherwise delivers different data
+        // properties
+        if (aop.getOperatorTag() != LogicalOperatorTag.GROUP
+                || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag())
+            return Boolean.FALSE;
+
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList();
+        GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg);
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyListsArg = gbyOpArg.getGroupByList();
+
+        List<Pair<LogicalVariable, ILogicalExpression>> listLeft = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
+        List<Pair<LogicalVariable, ILogicalExpression>> listRight = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
+
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyLists)
+            listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyListsArg)
+            listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
+
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight);
+
+        if (!isomorphic)
+            return Boolean.FALSE;
+        int sizeOp = op.getNestedPlans().size();
+        int sizeArg = gbyOpArg.getNestedPlans().size();
+        if (sizeOp != sizeArg)
+            return Boolean.FALSE;
+
+        GroupByOperator argOp = (GroupByOperator) arg;
+        List<ILogicalPlan> plans = op.getNestedPlans();
+        List<ILogicalPlan> plansArg = argOp.getNestedPlans();
+        for (int i = 0; i < plans.size(); i++) {
+            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
+            if (roots.size() != rootsArg.size())
+                return Boolean.FALSE;
+            for (int j = 0; j < roots.size(); j++) {
+                ILogicalOperator topOp1 = roots.get(j).getValue();
+                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
+                isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
+                if (!isomorphic)
+                    return Boolean.FALSE;
+            }
+        }
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.LIMIT)
+            return Boolean.FALSE;
+        LimitOperator limitOpArg = (LimitOperator) copyAndSubstituteVar(op, arg);
+        if (op.getOffset() != limitOpArg.getOffset())
+            return Boolean.FALSE;
+        boolean isomorphic = op.getMaxObjects().getValue().equals(limitOpArg.getMaxObjects().getValue());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.INNERJOIN)
+            return Boolean.FALSE;
+        InnerJoinOperator joinOpArg = (InnerJoinOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN)
+            return Boolean.FALSE;
+        LeftOuterJoinOperator joinOpArg = (LeftOuterJoinOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE)
+            return Boolean.FALSE;
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.ORDER)
+            return Boolean.FALSE;
+        OrderOperator orderOpArg = (OrderOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = compareIOrderAndExpressions(op.getOrderExpressions(), orderOpArg.getOrderExpressions());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.ASSIGN)
+            return Boolean.FALSE;
+        AssignOperator assignOpArg = (AssignOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(
+                getPairList(op.getVariables(), op.getExpressions()),
+                getPairList(assignOpArg.getVariables(), assignOpArg.getExpressions()));
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.SELECT)
+            return Boolean.FALSE;
+        SelectOperator selectOpArg = (SelectOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = op.getCondition().getValue().equals(selectOpArg.getCondition().getValue());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.PROJECT)
+            return Boolean.FALSE;
+        ProjectOperator projectOpArg = (ProjectOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), projectOpArg.getVariables());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.PARTITIONINGSPLIT)
+            return Boolean.FALSE;
+        PartitioningSplitOperator partitionOpArg = (PartitioningSplitOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = compareExpressions(op.getExpressions(), partitionOpArg.getExpressions());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.REPLICATE)
+            return Boolean.FALSE;
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.MATERIALIZE)
+            return Boolean.FALSE;
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.SCRIPT)
+            return Boolean.FALSE;
+        ScriptOperator scriptOpArg = (ScriptOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = op.getScriptDescription().equals(scriptOpArg.getScriptDescription());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.SUBPLAN)
+            return Boolean.FALSE;
+        SubplanOperator subplanOpArg = (SubplanOperator) copyAndSubstituteVar(op, arg);
+        List<ILogicalPlan> plans = op.getNestedPlans();
+        List<ILogicalPlan> plansArg = subplanOpArg.getNestedPlans();
+        for (int i = 0; i < plans.size(); i++) {
+            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
+            if (roots.size() == rootsArg.size())
+                return Boolean.FALSE;
+            for (int j = 0; j < roots.size(); j++) {
+                ILogicalOperator topOp1 = roots.get(j).getValue();
+                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
+                boolean isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
+                if (!isomorphic)
+                    return Boolean.FALSE;
+            }
+        }
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.UNIONALL)
+            return Boolean.FALSE;
+        UnionAllOperator unionOpArg = (UnionAllOperator) copyAndSubstituteVar(op, arg);
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mapping = op.getVariableMappings();
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mappingArg = unionOpArg.getVariableMappings();
+        if (mapping.size() != mappingArg.size())
+            return Boolean.FALSE;
+        return VariableUtilities.varListEqualUnordered(mapping, mappingArg);
+    }
+
+    @Override
+    public Boolean visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST)
+            return Boolean.FALSE;
+        UnnestOperator unnestOpArg = (UnnestOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables())
+                && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable());
+        if (!isomorphic)
+            return Boolean.FALSE;
+        isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST_MAP)
+            return Boolean.FALSE;
+        UnnestMapOperator unnestOpArg = (UnnestMapOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables());
+        if (!isomorphic)
+            return Boolean.FALSE;
+        isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
+            return Boolean.FALSE;
+        DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
+        if (!argScan.getDataSource().toString().equals(op.getDataSource().toString()))
+            return Boolean.FALSE;
+        DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), scanOpArg.getVariables())
+                && op.getDataSource().toString().equals(scanOpArg.getDataSource().toString());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.DISTINCT)
+            return Boolean.FALSE;
+        DistinctOperator distinctOpArg = (DistinctOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = compareExpressions(op.getExpressions(), distinctOpArg.getExpressions());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
+            return Boolean.FALSE;
+        // require the same partition property
+        if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag()))
+            return Boolean.FALSE;
+        variableMapping.clear();
+        IsomorphismUtilities.mapVariablesTopDown(op, arg, variableMapping);
+        IPhysicalPropertiesVector properties = op.getPhysicalOperator().getDeliveredProperties();
+        IPhysicalPropertiesVector propertiesArg = aop.getPhysicalOperator().getDeliveredProperties();
+        if (properties == null && propertiesArg == null)
+            return Boolean.TRUE;
+        if (properties == null || propertiesArg == null)
+            return Boolean.FALSE;
+        IPartitioningProperty partProp = properties.getPartitioningProperty();
+        IPartitioningProperty partPropArg = propertiesArg.getPartitioningProperty();
+        if (!partProp.getPartitioningType().equals(partPropArg.getPartitioningType()))
+            return Boolean.FALSE;
+        List<LogicalVariable> columns = new ArrayList<LogicalVariable>();
+        partProp.getColumns(columns);
+        List<LogicalVariable> columnsArg = new ArrayList<LogicalVariable>();
+        partPropArg.getColumns(columnsArg);
+        if (columns.size() != columnsArg.size())
+            return Boolean.FALSE;
+        if (columns.size() == 0)
+            return Boolean.TRUE;
+        for (int i = 0; i < columnsArg.size(); i++) {
+            LogicalVariable rightVar = columnsArg.get(i);
+            LogicalVariable leftVar = variableMapping.get(rightVar);
+            if (leftVar != null)
+                columnsArg.set(i, leftVar);
+        }
+        return VariableUtilities.varListEqualUnordered(columns, columnsArg);
+    }
+
+    @Override
+    public Boolean visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.WRITE)
+            return Boolean.FALSE;
+        WriteOperator writeOpArg = (WriteOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitDistributeResultOperator(DistributeResultOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT)
+            return Boolean.FALSE;
+        DistributeResultOperator writeOpArg = (DistributeResultOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT)
+            return Boolean.FALSE;
+        WriteResultOperator writeOpArg = (WriteResultOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
+        if (!op.getDataSource().equals(writeOpArg.getDataSource()))
+            isomorphic = false;
+        if (!op.getPayloadExpression().equals(writeOpArg.getPayloadExpression()))
+            isomorphic = false;
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitInsertDeleteOperator(InsertDeleteOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE)
+            return Boolean.FALSE;
+        InsertDeleteOperator insertOpArg = (InsertDeleteOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema());
+        if (!op.getDataSource().equals(insertOpArg.getDataSource()))
+            isomorphic = false;
+        if (!op.getPayloadExpression().equals(insertOpArg.getPayloadExpression()))
+            isomorphic = false;
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.INDEX_INSERT_DELETE)
+            return Boolean.FALSE;
+        IndexInsertDeleteOperator insertOpArg = (IndexInsertDeleteOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema());
+        if (!op.getDataSourceIndex().equals(insertOpArg.getDataSourceIndex()))
+            isomorphic = false;
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitTokenizeOperator(TokenizeOperator op, ILogicalOperator arg) throws AlgebricksException {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+        if (aop.getOperatorTag() != LogicalOperatorTag.TOKENIZE)
+            return Boolean.FALSE;
+        TokenizeOperator tokenizeOpArg = (TokenizeOperator) copyAndSubstituteVar(op, arg);
+        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), tokenizeOpArg.getSchema());
+        if (!op.getDataSourceIndex().equals(tokenizeOpArg.getDataSourceIndex()))
+            isomorphic = false;
+        return isomorphic;
+    }
+
+    @Override
+    public Boolean visitSinkOperator(SinkOperator op, ILogicalOperator arg) throws AlgebricksException {
+        return true;
+    }
+
+    private Boolean compareExpressions(List<Mutable<ILogicalExpression>> opExprs,
+            List<Mutable<ILogicalExpression>> argExprs) {
+        if (opExprs.size() != argExprs.size())
+            return Boolean.FALSE;
+        for (int i = 0; i < opExprs.size(); i++) {
+            boolean isomorphic = opExprs.get(i).getValue().equals(argExprs.get(i).getValue());
+            if (!isomorphic)
+                return Boolean.FALSE;
+        }
+        return Boolean.TRUE;
+    }
+
+    @Override
+    public Boolean visitExternalDataLookupOperator(ExternalDataLookupOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        return Boolean.FALSE;
+    }
+
+    private Boolean compareIOrderAndExpressions(List<Pair<IOrder, Mutable<ILogicalExpression>>> opOrderExprs,
+            List<Pair<IOrder, Mutable<ILogicalExpression>>> argOrderExprs) {
+        if (opOrderExprs.size() != argOrderExprs.size())
+            return Boolean.FALSE;
+        for (int i = 0; i < opOrderExprs.size(); i++) {
+            boolean isomorphic = opOrderExprs.get(i).first.equals(argOrderExprs.get(i).first);
+            if (!isomorphic)
+                return Boolean.FALSE;
+            isomorphic = opOrderExprs.get(i).second.getValue().equals(argOrderExprs.get(i).second.getValue());
+            if (!isomorphic)
+                return Boolean.FALSE;
+        }
+        return Boolean.TRUE;
+    }
+
+    private ILogicalOperator copyAndSubstituteVar(ILogicalOperator op, ILogicalOperator argOp)
+            throws AlgebricksException {
+        ILogicalOperator newOp = OperatorManipulationUtil.deepCopy(argOp);
+        variableMapping.clear();
+        IsomorphismUtilities.mapVariablesTopDown(op, argOp, variableMapping);
+
+        List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+        if (argOp.getInputs().size() > 0)
+            for (int i = 0; i < argOp.getInputs().size(); i++)
+                VariableUtilities.getLiveVariables(argOp.getInputs().get(i).getValue(), liveVars);
+        List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(argOp, producedVars);
+        List<LogicalVariable> producedVarsNew = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(op, producedVarsNew);
+
+        if (producedVars.size() != producedVarsNew.size())
+            return newOp;
+        for (Entry<LogicalVariable, LogicalVariable> map : variableMapping.entrySet()) {
+            if (liveVars.contains(map.getKey())) {
+                VariableUtilities.substituteVariables(newOp, map.getKey(), map.getValue(), null);
+            }
+        }
+        for (int i = 0; i < producedVars.size(); i++)
+            VariableUtilities.substituteVariables(newOp, producedVars.get(i), producedVarsNew.get(i), null);
+        return newOp;
+    }
+
+    public List<Pair<LogicalVariable, ILogicalExpression>> getPairList(List<LogicalVariable> vars,
+            List<Mutable<ILogicalExpression>> exprs) throws AlgebricksException {
+        List<Pair<LogicalVariable, ILogicalExpression>> list = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
+        if (vars.size() != exprs.size())
+            throw new AlgebricksException("variable list size does not equal to expression list size ");
+        for (int i = 0; i < vars.size(); i++) {
+            list.add(new Pair<LogicalVariable, ILogicalExpression>(vars.get(i), exprs.get(i).getValue()));
+        }
+        return list;
+    }
+
+    private static boolean variableEqual(LogicalVariable var, LogicalVariable varArg) {
+        if (var == null && varArg == null)
+            return true;
+        if (var.equals(varArg))
+            return true;
+        else
+            return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
new file mode 100644
index 0000000..d79853d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class IsomorphismUtilities {
+
+    public static void mapVariablesTopDown(ILogicalOperator op, ILogicalOperator arg,
+            Map<LogicalVariable, LogicalVariable> variableMapping) throws AlgebricksException {
+        IsomorphismVariableMappingVisitor visitor = new IsomorphismVariableMappingVisitor(variableMapping);
+        op.accept(visitor, arg);
+    }
+
+    public static boolean isOperatorIsomorphic(ILogicalOperator op, ILogicalOperator arg) throws AlgebricksException {
+        IsomorphismOperatorVisitor visitor = new IsomorphismOperatorVisitor();
+        return op.accept(visitor, arg).booleanValue();
+    }
+
+    public static boolean isOperatorIsomorphicPlanSegment(ILogicalOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> inputs1 = op.getInputs();
+        List<Mutable<ILogicalOperator>> inputs2 = arg.getInputs();
+        if (inputs1.size() != inputs2.size())
+            return Boolean.FALSE;
+        for (int i = 0; i < inputs1.size(); i++) {
+            ILogicalOperator input1 = inputs1.get(i).getValue();
+            ILogicalOperator input2 = inputs2.get(i).getValue();
+            boolean isomorphic = isOperatorIsomorphicPlanSegment(input1, input2);
+            if (!isomorphic)
+                return Boolean.FALSE;
+        }
+        return IsomorphismUtilities.isOperatorIsomorphic(op, arg);
+    }
+
+    public static boolean isOperatorIsomorphicPlan(ILogicalPlan plan, ILogicalPlan arg) throws AlgebricksException {
+        if (plan.getRoots().size() != arg.getRoots().size()) {
+            return false;
+        }
+        for (int i = 0; i < plan.getRoots().size(); i++) {
+            ILogicalOperator topOp1 = plan.getRoots().get(i).getValue();
+            ILogicalOperator topOp2 = arg.getRoots().get(i).getValue();
+            if (!IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}

[03/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java
deleted file mode 100644
index 46083bd..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunctionFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-
-public interface ICopyUnnestingFunctionFactory extends Serializable {
-    public ICopyUnnestingFunction createUnnestingFunction(IDataOutputProvider provider) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java
deleted file mode 100644
index ff0c429..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntime.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public interface IPushRuntime extends IFrameWriter {
-    public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc);
-
-    public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java
deleted file mode 100644
index 703bc1f..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IPushRuntimeFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface IPushRuntimeFactory extends Serializable {
-    public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException, HyracksDataException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java
deleted file mode 100644
index 8f1fb23..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface IRunningAggregateEvaluator {
-    public void init() throws AlgebricksException;
-
-    public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java
deleted file mode 100644
index 96cdc2f..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IRunningAggregateEvaluatorFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public interface IRunningAggregateEvaluatorFactory extends Serializable {
-    public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java
deleted file mode 100644
index 2480422..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluator.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface IScalarEvaluator {
-    public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java
deleted file mode 100644
index 037b8ed..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IScalarEvaluatorFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface IScalarEvaluatorFactory extends Serializable {
-    public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java
deleted file mode 100644
index e102605..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface IUnnestingEvaluator {
-    public void init(IFrameTupleReference tuple) throws AlgebricksException;
-
-    public boolean step(IPointable result) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java
deleted file mode 100644
index 02bc7a7..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingEvaluatorFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface IUnnestingEvaluatorFactory extends Serializable {
-    public IUnnestingEvaluator createUnnestingEvaluator(IHyracksTaskContext ctx) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingPositionWriter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingPositionWriter.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingPositionWriter.java
deleted file mode 100644
index 65f5083..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IUnnestingPositionWriter.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-public interface IUnnestingPositionWriter {
-    public void write(DataOutput dataOutput, long position) throws IOException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java
deleted file mode 100644
index 1ba4bdf..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ColumnAccessEvalFactory.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class ColumnAccessEvalFactory implements ICopyEvaluatorFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final int fieldIndex;
-
-    public ColumnAccessEvalFactory(int fieldIndex) {
-        this.fieldIndex = fieldIndex;
-    }
-
-    @Override
-    public String toString() {
-        return "ColumnAccess(" + fieldIndex + ")";
-    }
-
-    @Override
-    public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
-        return new ICopyEvaluator() {
-
-            private DataOutput out = output.getDataOutput();
-
-            @Override
-            public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
-                byte[] buffer = tuple.getFieldData(fieldIndex);
-                int start = tuple.getFieldStart(fieldIndex);
-                int length = tuple.getFieldLength(fieldIndex);
-                try {
-                    out.write(buffer, start, length);
-                } catch (IOException ioe) {
-                    throw new AlgebricksException(ioe);
-                }
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java
deleted file mode 100644
index e637338..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvalFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class ConstantEvalFactory implements ICopyEvaluatorFactory {
-    private static final long serialVersionUID = 1L;
-
-    private byte[] value;
-
-    public ConstantEvalFactory(byte[] value) {
-        this.value = value;
-    }
-
-    @Override
-    public String toString() {
-        return "Constant";
-    }
-
-    @Override
-    public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
-        return new ICopyEvaluator() {
-
-            private DataOutput out = output.getDataOutput();
-
-            @Override
-            public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
-                try {
-                    out.write(value, 0, value.length);
-                } catch (IOException ioe) {
-                    throw new AlgebricksException(ioe);
-                }
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java
deleted file mode 100644
index ecfd48d..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/ConstantEvaluatorFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class ConstantEvaluatorFactory implements IScalarEvaluatorFactory {
-    private static final long serialVersionUID = 1L;
-
-    private byte[] value;
-
-    public ConstantEvaluatorFactory(byte[] value) {
-        this.value = value;
-    }
-
-    @Override
-    public String toString() {
-        return "Constant";
-    }
-
-    @Override
-    public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
-        return new IScalarEvaluator() {
-            @Override
-            public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
-                result.set(value, 0, value.length);
-            }
-        };
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java
deleted file mode 100644
index c84ca26..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/evaluators/TupleFieldEvaluatorFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.evaluators;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class TupleFieldEvaluatorFactory implements IScalarEvaluatorFactory {
-    private static final long serialVersionUID = 1L;
-
-    private final int fieldIndex;
-
-    public TupleFieldEvaluatorFactory(int fieldIndex) {
-        this.fieldIndex = fieldIndex;
-    }
-
-    @Override
-    public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
-        return new IScalarEvaluator() {
-            @Override
-            public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
-                result.set(tuple.getFieldData(fieldIndex), tuple.getFieldStart(fieldIndex),
-                        tuple.getFieldLength(fieldIndex));
-            }
-        };
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
deleted file mode 100644
index e00e64a..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/AggregateRuntimeFactory.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-
-public class AggregateRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    // private int[] outColumns;
-    private IAggregateEvaluatorFactory[] aggregFactories;
-
-    public AggregateRuntimeFactory(IAggregateEvaluatorFactory[] aggregFactories) {
-        super(null);
-        // this.outColumns = outColumns;
-        this.aggregFactories = aggregFactories;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("assign [");
-        for (int i = 0; i < aggregFactories.length; i++) {
-            if (i > 0) {
-                sb.append(", ");
-            }
-            sb.append(aggregFactories[i]);
-        }
-        sb.append("]");
-        return sb.toString();
-    }
-
-    @Override
-    public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException {
-        return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
-            private IAggregateEvaluator[] aggregs = new IAggregateEvaluator[aggregFactories.length];
-            private IPointable result = VoidPointable.FACTORY.createPointable();
-            private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(aggregs.length);
-
-            private boolean first = true;
-
-            @Override
-            public void open() throws HyracksDataException {
-                try {
-                    if (first) {
-                        first = false;
-                        initAccessAppendRef(ctx);
-                        for (int i = 0; i < aggregFactories.length; i++) {
-                            aggregs[i] = aggregFactories[i].createAggregateEvaluator(ctx);
-                        }
-                    }
-                    for (int i = 0; i < aggregFactories.length; i++) {
-                        aggregs[i].init();
-                    }
-                } catch (AlgebricksException e) {
-                    throw new HyracksDataException(e);
-                }
-
-                writer.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-                for (int t = 0; t < nTuple; t++) {
-                    tRef.reset(tAccess, t);
-                    processTuple(tRef);
-                }
-
-            }
-
-            @Override
-            public void close() throws HyracksDataException {
-                computeAggregate();
-                appendToFrameFromTupleBuilder(tupleBuilder);
-                super.close();
-            }
-
-            private void computeAggregate() throws HyracksDataException {
-                tupleBuilder.reset();
-                for (int f = 0; f < aggregs.length; f++) {
-                    try {
-                        aggregs[f].finish(result);
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                    tupleBuilder.addField(result.getByteArray(), result.getStartOffset(), result.getLength());
-                }
-            }
-
-            private void processTuple(FrameTupleReference tupleRef) throws HyracksDataException {
-                for (int f = 0; f < aggregs.length; f++) {
-                    try {
-                        aggregs[f].step(tupleRef);
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-                writer.fail();
-            }
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
deleted file mode 100644
index 618768c..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.std.group.AbstractAccumulatingAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
-
-public class NestedPlansAccumulatingAggregatorFactory extends AbstractAccumulatingAggregatorDescriptorFactory {
-
-    private static final long serialVersionUID = 1L;
-    private AlgebricksPipeline[] subplans;
-    private int[] keyFieldIdx;
-    private int[] decorFieldIdx;
-
-    public NestedPlansAccumulatingAggregatorFactory(AlgebricksPipeline[] subplans, int[] keyFieldIdx,
-            int[] decorFieldIdx) {
-        this.subplans = subplans;
-        this.keyFieldIdx = keyFieldIdx;
-        this.decorFieldIdx = decorFieldIdx;
-    }
-
-    @Override
-    public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
-            RecordDescriptor outRecordDescriptor, int[] keys, int[] partialKeys) throws HyracksDataException {
-
-        final AggregatorOutput outputWriter = new AggregatorOutput(subplans, keyFieldIdx.length,
-                decorFieldIdx.length);
-        final NestedTupleSourceRuntime[] pipelines = new NestedTupleSourceRuntime[subplans.length];
-        for (int i = 0; i < subplans.length; i++) {
-            try {
-                pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, ctx);
-            } catch (AlgebricksException e) {
-                throw new HyracksDataException(e);
-            }
-        }
-
-        return new IAggregatorDescriptor() {
-
-            @Override
-            public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                ArrayTupleBuilder tb = outputWriter.getTupleBuilder();
-                tb.reset();
-                for (int i = 0; i < keyFieldIdx.length; ++i) {
-                    tb.addField(accessor, tIndex, keyFieldIdx[i]);
-                }
-                for (int i = 0; i < decorFieldIdx.length; ++i) {
-                    tb.addField(accessor, tIndex, decorFieldIdx[i]);
-                }
-                for (int i = 0; i < pipelines.length; ++i) {
-                    pipelines[i].open();
-                }
-
-                // aggregate the first tuple
-                for (int i = 0; i < pipelines.length; i++) {
-                    pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
-                }
-            }
-
-            @Override
-            public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
-                    int stateTupleIndex, AggregateState state) throws HyracksDataException {
-                // it only works if the output of the aggregator fits in one
-                // frame
-                for (int i = 0; i < pipelines.length; i++) {
-                    pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
-                }
-            }
-
-            @Override
-            public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                for (int i = 0; i < pipelines.length; i++) {
-                    outputWriter.setInputIdx(i);
-                    pipelines[i].close();
-                }
-                // outputWriter.writeTuple(appender);
-                tupleBuilder.reset();
-                ArrayTupleBuilder tb = outputWriter.getTupleBuilder();
-                byte[] data = tb.getByteArray();
-                int[] fieldEnds = tb.getFieldEndOffsets();
-                int start = 0;
-                int offset = 0;
-                for (int i = 0; i < fieldEnds.length; i++) {
-                    if (i > 0)
-                        start = fieldEnds[i - 1];
-                    offset = fieldEnds[i] - start;
-                    tupleBuilder.addField(data, start, offset);
-                }
-                return true;
-            }
-
-            @Override
-            public AggregateState createAggregateStates() {
-                return new AggregateState();
-            }
-
-            @Override
-            public void reset() {
-
-            }
-
-            @Override
-            public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor,
-                    int tIndex, AggregateState state) throws HyracksDataException {
-                throw new IllegalStateException("this method should not be called");
-            }
-
-            @Override
-            public void close() {
-
-            }
-
-        };
-    }
-
-    private IFrameWriter assemblePipeline(AlgebricksPipeline subplan, IFrameWriter writer, IHyracksTaskContext ctx)
-            throws AlgebricksException, HyracksDataException {
-        // plug the operators
-        IFrameWriter start = writer;
-        IPushRuntimeFactory[] runtimeFactories = subplan.getRuntimeFactories();
-        RecordDescriptor[] recordDescriptors = subplan.getRecordDescriptors();
-        for (int i = runtimeFactories.length - 1; i >= 0; i--) {
-            IPushRuntime newRuntime = runtimeFactories[i].createPushRuntime(ctx);
-            newRuntime.setFrameWriter(0, start, recordDescriptors[i]);
-            if (i > 0) {
-                newRuntime.setInputRecordDescriptor(0, recordDescriptors[i - 1]);
-            } else {
-                // the nts has the same input and output rec. desc.
-                newRuntime.setInputRecordDescriptor(0, recordDescriptors[0]);
-            }
-            start = newRuntime;
-        }
-        return start;
-    }
-
-    /**
-     * We suppose for now, that each subplan only produces one tuple.
-     */
-    private static class AggregatorOutput implements IFrameWriter {
-
-        // private ByteBuffer frame;
-        private FrameTupleAccessor[] tAccess;
-        private RecordDescriptor[] inputRecDesc;
-        private int inputIdx;
-        private ArrayTupleBuilder tb;
-        private AlgebricksPipeline[] subplans;
-
-        public AggregatorOutput(AlgebricksPipeline[] subplans, int numKeys, int numDecors) {
-            this.subplans = subplans;
-            // this.keyFieldIndexes = keyFieldIndexes;
-            int totalAggFields = 0;
-            this.inputRecDesc = new RecordDescriptor[subplans.length];
-            for (int i = 0; i < subplans.length; i++) {
-                RecordDescriptor[] rd = subplans[i].getRecordDescriptors();
-                this.inputRecDesc[i] = rd[rd.length - 1];
-                totalAggFields += subplans[i].getOutputWidth();
-            }
-            tb = new ArrayTupleBuilder(numKeys + numDecors + totalAggFields);
-
-            this.tAccess = new FrameTupleAccessor[inputRecDesc.length];
-            for (int i = 0; i < inputRecDesc.length; i++) {
-                tAccess[i] = new FrameTupleAccessor(inputRecDesc[i]);
-            }
-        }
-
-        @Override
-        public void open() throws HyracksDataException {
-
-        }
-
-        /**
-         * Since each pipeline only produces one tuple, this method is only
-         * called by the close method of the pipelines.
-         */
-        @Override
-        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-            int tIndex = 0;
-            int w = subplans[inputIdx].getOutputWidth();
-            IFrameTupleAccessor accessor = tAccess[inputIdx];
-            accessor.reset(buffer);
-            for (int f = 0; f < w; f++) {
-                tb.addField(accessor, tIndex, f);
-            }
-        }
-
-        @Override
-        public void close() throws HyracksDataException {
-            // clearFrame();
-        }
-
-        public void setInputIdx(int inputIdx) {
-            this.inputIdx = inputIdx;
-        }
-
-        public ArrayTupleBuilder getTupleBuilder() {
-            return tb;
-        }
-
-        @Override
-        public void fail() throws HyracksDataException {
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java
deleted file mode 100644
index b7e736e..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/NestedPlansRunningAggregatorFactory.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.comm.VSizeFrame;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-
-public class NestedPlansRunningAggregatorFactory implements IAggregatorDescriptorFactory {
-
-    private static final long serialVersionUID = 1L;
-    private final AlgebricksPipeline[] subplans;
-    private final int[] keyFieldIdx;
-    private final int[] decorFieldIdx;
-
-    public NestedPlansRunningAggregatorFactory(AlgebricksPipeline[] subplans, int[] keyFieldIdx, int[] decorFieldIdx) {
-        this.subplans = subplans;
-        this.keyFieldIdx = keyFieldIdx;
-        this.decorFieldIdx = decorFieldIdx;
-    }
-
-    /* (non-Javadoc)
-     * @see edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory#createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext, edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor, edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor, int[], int[])
-     */
-    @Override
-    public IAggregatorDescriptor createAggregator(final IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
-            RecordDescriptor outRecordDescriptor, int[] keyFields, int[] keyFieldsInPartialResults,
-            final IFrameWriter writer) throws HyracksDataException {
-        final RunningAggregatorOutput outputWriter = new RunningAggregatorOutput(ctx, subplans, keyFieldIdx.length,
-                decorFieldIdx.length, writer);
-        final NestedTupleSourceRuntime[] pipelines = new NestedTupleSourceRuntime[subplans.length];
-        for (int i = 0; i < subplans.length; i++) {
-            try {
-                pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, ctx);
-            } catch (AlgebricksException e) {
-                throw new HyracksDataException(e);
-            }
-        }
-
-        final ArrayTupleBuilder gbyTb = outputWriter.getGroupByTupleBuilder();
-
-        return new IAggregatorDescriptor() {
-
-            @Override
-            public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-
-                for (int i = 0; i < pipelines.length; ++i) {
-                    pipelines[i].open();
-                }
-
-                gbyTb.reset();
-                for (int i = 0; i < keyFieldIdx.length; ++i) {
-                    gbyTb.addField(accessor, tIndex, keyFieldIdx[i]);
-                }
-                for (int i = 0; i < decorFieldIdx.length; ++i) {
-                    gbyTb.addField(accessor, tIndex, decorFieldIdx[i]);
-                }
-
-                // aggregate the first tuple
-                for (int i = 0; i < pipelines.length; i++) {
-                    outputWriter.setInputIdx(i);
-                    pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
-                }
-            }
-
-            @Override
-            public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
-                    int stateTupleIndex, AggregateState state) throws HyracksDataException {
-                for (int i = 0; i < pipelines.length; i++) {
-                    outputWriter.setInputIdx(i);
-                    pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
-                }
-            }
-
-            @Override
-            public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                for (int i = 0; i < pipelines.length; ++i) {
-                    outputWriter.setInputIdx(i);
-                    pipelines[i].close();
-                }
-                return false;
-            }
-
-            @Override
-            public AggregateState createAggregateStates() {
-                return new AggregateState();
-            }
-
-            @Override
-            public void reset() {
-
-            }
-
-            @Override
-            public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor,
-                    int tIndex, AggregateState state) throws HyracksDataException {
-                throw new IllegalStateException("this method should not be called");
-            }
-
-            @Override
-            public void close() {
-
-            }
-        };
-    }
-
-    private IFrameWriter assemblePipeline(AlgebricksPipeline subplan, IFrameWriter writer, IHyracksTaskContext ctx)
-            throws AlgebricksException, HyracksDataException {
-        // plug the operators
-        IFrameWriter start = writer;
-        IPushRuntimeFactory[] runtimeFactories = subplan.getRuntimeFactories();
-        RecordDescriptor[] recordDescriptors = subplan.getRecordDescriptors();
-        for (int i = runtimeFactories.length - 1; i >= 0; i--) {
-            IPushRuntime newRuntime = runtimeFactories[i].createPushRuntime(ctx);
-            newRuntime.setFrameWriter(0, start, recordDescriptors[i]);
-            if (i > 0) {
-                newRuntime.setInputRecordDescriptor(0, recordDescriptors[i - 1]);
-            } else {
-                // the nts has the same input and output rec. desc.
-                newRuntime.setInputRecordDescriptor(0, recordDescriptors[0]);
-            }
-            start = newRuntime;
-        }
-        return start;
-    }
-
-    private static class RunningAggregatorOutput implements IFrameWriter {
-
-        private final FrameTupleAccessor[] tAccess;
-        private final RecordDescriptor[] inputRecDesc;
-        private int inputIdx;
-        private final ArrayTupleBuilder tb;
-        private final ArrayTupleBuilder gbyTb;
-        private final AlgebricksPipeline[] subplans;
-        private final IFrameWriter outputWriter;
-        private final FrameTupleAppender outputAppender;
-
-        public RunningAggregatorOutput(IHyracksTaskContext ctx, AlgebricksPipeline[] subplans, int numKeys,
-                int numDecors, IFrameWriter outputWriter) throws HyracksDataException {
-            this.subplans = subplans;
-            this.outputWriter = outputWriter;
-
-            // this.keyFieldIndexes = keyFieldIndexes;
-            int totalAggFields = 0;
-            this.inputRecDesc = new RecordDescriptor[subplans.length];
-            for (int i = 0; i < subplans.length; i++) {
-                RecordDescriptor[] rd = subplans[i].getRecordDescriptors();
-                this.inputRecDesc[i] = rd[rd.length - 1];
-                totalAggFields += subplans[i].getOutputWidth();
-            }
-            tb = new ArrayTupleBuilder(numKeys + numDecors + totalAggFields);
-            gbyTb = new ArrayTupleBuilder(numKeys + numDecors);
-
-            this.tAccess = new FrameTupleAccessor[inputRecDesc.length];
-            for (int i = 0; i < inputRecDesc.length; i++) {
-                tAccess[i] = new FrameTupleAccessor(inputRecDesc[i]);
-            }
-
-            this.outputAppender = new FrameTupleAppender(new VSizeFrame(ctx));
-        }
-
-        @Override
-        public void open() throws HyracksDataException {
-
-        }
-
-        @Override
-        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-            int w = subplans[inputIdx].getOutputWidth();
-            IFrameTupleAccessor accessor = tAccess[inputIdx];
-            accessor.reset(buffer);
-            for (int tIndex = 0; tIndex < accessor.getTupleCount(); tIndex++) {
-                tb.reset();
-                byte[] data = gbyTb.getByteArray();
-                int[] fieldEnds = gbyTb.getFieldEndOffsets();
-                int start = 0;
-                int offset = 0;
-                for (int i = 0; i < fieldEnds.length; i++) {
-                    if (i > 0)
-                        start = fieldEnds[i - 1];
-                    offset = fieldEnds[i] - start;
-                    tb.addField(data, start, offset);
-                }
-                for (int f = 0; f < w; f++) {
-                    tb.addField(accessor, tIndex, f);
-                }
-                FrameUtils.appendToWriter(outputWriter, outputAppender, tb.getFieldEndOffsets(),
-                        tb.getByteArray(), 0, tb.getSize());
-            }
-        }
-
-        @Override
-        public void close() throws HyracksDataException {
-            outputAppender.flush(outputWriter, true);
-        }
-
-        public void setInputIdx(int inputIdx) {
-            this.inputIdx = inputIdx;
-        }
-
-        public ArrayTupleBuilder getGroupByTupleBuilder() {
-            return gbyTb;
-        }
-
-        @Override
-        public void fail() throws HyracksDataException {
-        }
-
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
deleted file mode 100644
index 39db3e1..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
-
-import java.io.DataOutput;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.std.group.AbstractAccumulatingAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
-
-public class SerializableAggregatorDescriptorFactory extends AbstractAccumulatingAggregatorDescriptorFactory {
-    private static final long serialVersionUID = 1L;
-    private ICopySerializableAggregateFunctionFactory[] aggFactories;
-
-    public SerializableAggregatorDescriptorFactory(ICopySerializableAggregateFunctionFactory[] aggFactories) {
-        this.aggFactories = aggFactories;
-    }
-
-    @Override
-    public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
-            RecordDescriptor outRecordDescriptor, int[] keyFields, final int[] keyFieldsInPartialResults)
-            throws HyracksDataException {
-        final int[] keys = keyFields;
-
-        /**
-         * one IAggregatorDescriptor instance per Gby operator
-         */
-        return new IAggregatorDescriptor() {
-            private FrameTupleReference ftr = new FrameTupleReference();
-            private ICopySerializableAggregateFunction[] aggs = new ICopySerializableAggregateFunction[aggFactories.length];
-            private int offsetFieldIndex = keys.length;
-            private int stateFieldLength[] = new int[aggFactories.length];
-
-            @Override
-            public AggregateState createAggregateStates() {
-                return new AggregateState();
-            }
-
-            @Override
-            public void init(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex, AggregateState state)
-                    throws HyracksDataException {
-                DataOutput output = tb.getDataOutput();
-                ftr.reset(accessor, tIndex);
-                for (int i = 0; i < aggs.length; i++) {
-                    try {
-                        int begin = tb.getSize();
-                        if (aggs[i] == null) {
-                            aggs[i] = aggFactories[i].createAggregateFunction();
-                        }
-                        aggs[i].init(output);
-                        tb.addFieldEndOffset();
-                        stateFieldLength[i] = tb.getSize() - begin;
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-
-                // doing initial aggregate
-                ftr.reset(accessor, tIndex);
-                for (int i = 0; i < aggs.length; i++) {
-                    try {
-                        byte[] data = tb.getByteArray();
-                        int prevFieldPos = i + keys.length - 1;
-                        int start = prevFieldPos >= 0 ? tb.getFieldEndOffsets()[prevFieldPos] : 0;
-                        aggs[i].step(ftr, data, start, stateFieldLength[i]);
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-            }
-
-            @Override
-            public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
-                    int stateTupleIndex, AggregateState state) throws HyracksDataException {
-                ftr.reset(accessor, tIndex);
-                int stateTupleStart = stateAccessor.getTupleStartOffset(stateTupleIndex);
-                int fieldSlotLength = stateAccessor.getFieldSlotsLength();
-                for (int i = 0; i < aggs.length; i++) {
-                    try {
-                        byte[] data = stateAccessor.getBuffer().array();
-                        int start = stateAccessor.getFieldStartOffset(stateTupleIndex, i + keys.length)
-                                + stateTupleStart + fieldSlotLength;
-                        aggs[i].step(ftr, data, start, stateFieldLength[i]);
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-            }
-
-            @Override
-            public boolean outputPartialResult(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                byte[] data = accessor.getBuffer().array();
-                int startOffset = accessor.getTupleStartOffset(tIndex);
-                int aggFieldOffset = accessor.getFieldStartOffset(tIndex, offsetFieldIndex);
-                int refOffset = startOffset + accessor.getFieldSlotsLength() + aggFieldOffset;
-                int start = refOffset;
-                for (int i = 0; i < aggs.length; i++) {
-                    try {
-                        aggs[i].finishPartial(data, start, stateFieldLength[i], tb.getDataOutput());
-                        start += stateFieldLength[i];
-                        tb.addFieldEndOffset();
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-                return true;
-            }
-
-            @Override
-            public boolean outputFinalResult(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                byte[] data = accessor.getBuffer().array();
-                int startOffset = accessor.getTupleStartOffset(tIndex);
-                int aggFieldOffset = accessor.getFieldStartOffset(tIndex, offsetFieldIndex);
-                int refOffset = startOffset + accessor.getFieldSlotsLength() + aggFieldOffset;
-                int start = refOffset;
-                for (int i = 0; i < aggs.length; i++) {
-                    try {
-                        aggs[i].finish(data, start, stateFieldLength[i], tb.getDataOutput());
-                        start += stateFieldLength[i];
-                        tb.addFieldEndOffset();
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-                return true;
-            }
-
-            @Override
-            public void reset() {
-
-            }
-
-            @Override
-            public void close() {
-                reset();
-            }
-
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
deleted file mode 100644
index b3eab7b..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.std.group.AbstractAccumulatingAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
-
-public class SimpleAlgebricksAccumulatingAggregatorFactory extends AbstractAccumulatingAggregatorDescriptorFactory {
-
-    private static final long serialVersionUID = 1L;
-    private IAggregateEvaluatorFactory[] aggFactories;
-
-    public SimpleAlgebricksAccumulatingAggregatorFactory(IAggregateEvaluatorFactory[] aggFactories, int[] keys) {
-        this.aggFactories = aggFactories;
-    }
-
-    @Override
-    public IAggregatorDescriptor createAggregator(final IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
-            RecordDescriptor outRecordDescriptor, int[] aggKeys, int[] partialKeys) throws HyracksDataException {
-
-        return new IAggregatorDescriptor() {
-
-            private FrameTupleReference ftr = new FrameTupleReference();
-            private IPointable p = VoidPointable.FACTORY.createPointable();
-
-            @Override
-            public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
-
-                // initialize aggregate functions
-                for (int i = 0; i < agg.length; i++) {
-                    try {
-                        agg[i].init();
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-
-                ftr.reset(accessor, tIndex);
-                for (int i = 0; i < agg.length; i++) {
-                    try {
-                        agg[i].step(ftr);
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-            }
-
-            @Override
-            public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
-                    int stateTupleIndex, AggregateState state) throws HyracksDataException {
-                IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
-                ftr.reset(accessor, tIndex);
-                for (int i = 0; i < agg.length; i++) {
-                    try {
-                        agg[i].step(ftr);
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-            }
-
-            @Override
-            public boolean outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
-                    AggregateState state) throws HyracksDataException {
-                IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
-                for (int i = 0; i < agg.length; i++) {
-                    try {
-                        agg[i].finish(p);
-                        tupleBuilder.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-                return true;
-            }
-
-            @Override
-            public AggregateState createAggregateStates() {
-                IAggregateEvaluator[] agg = new IAggregateEvaluator[aggFactories.length];
-                for (int i = 0; i < agg.length; i++) {
-                    try {
-                        agg[i] = aggFactories[i].createAggregateEvaluator(ctx);
-                    } catch (AlgebricksException e) {
-                        throw new IllegalStateException(e);
-                    }
-                }
-                return new AggregateState(agg);
-            }
-
-            @Override
-            public void reset() {
-
-            }
-
-            @Override
-            public boolean outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor,
-                    int tIndex, AggregateState state) throws HyracksDataException {
-                IAggregateEvaluator[] agg = (IAggregateEvaluator[]) state.state;
-                for (int i = 0; i < agg.length; i++) {
-                    try {
-                        agg[i].finishPartial(p);
-                        tupleBuilder.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
-                    } catch (AlgebricksException e) {
-                        throw new HyracksDataException(e);
-                    }
-                }
-                return true;
-            }
-
-            @Override
-            public void close() {
-
-            }
-
-        };
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFieldFramePushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFieldFramePushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFieldFramePushRuntime.java
deleted file mode 100644
index cd71125..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFieldFramePushRuntime.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import edu.uci.ics.hyracks.api.comm.IFrameFieldAppender;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAppender;
-import edu.uci.ics.hyracks.api.comm.VSizeFrame;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameFixedFieldTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-
-public abstract class AbstractOneInputOneOutputOneFieldFramePushRuntime
-        extends AbstractOneInputOneOutputOneFramePushRuntime {
-
-    @Override
-    protected IFrameTupleAppender getTupleAppender() {
-        return (FrameFixedFieldTupleAppender) appender;
-    }
-
-    protected IFrameFieldAppender getFieldAppender() {
-        return (FrameFixedFieldTupleAppender) appender;
-    }
-
-    protected final void initAccessAppendFieldRef(IHyracksTaskContext ctx) throws HyracksDataException {
-        frame = new VSizeFrame(ctx);
-        appender = new FrameFixedFieldTupleAppender(inputRecordDesc.getFieldCount());
-        appender.reset(frame, true);
-        tAccess = new FrameTupleAccessor(inputRecordDesc);
-        tRef = new FrameTupleReference();
-    }
-
-    protected void appendField(byte[] array, int start, int length) throws HyracksDataException {
-        FrameUtils.appendFieldToWriter(writer, getFieldAppender(), array, start, length);
-    }
-
-    protected void appendField(IFrameTupleAccessor accessor, int tid, int fid) throws HyracksDataException {
-        FrameUtils.appendFieldToWriter(writer, getFieldAppender(), accessor, tid, fid);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
deleted file mode 100644
index ec4e039..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import edu.uci.ics.hyracks.api.comm.IFrame;
-import edu.uci.ics.hyracks.api.comm.IFrameAppender;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAppender;
-import edu.uci.ics.hyracks.api.comm.VSizeFrame;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-
-public abstract class AbstractOneInputOneOutputOneFramePushRuntime extends AbstractOneInputOneOutputPushRuntime {
-
-    protected IFrameAppender appender;
-    protected IFrame frame;
-    protected FrameTupleAccessor tAccess;
-    protected FrameTupleReference tRef;
-
-    protected final void initAccessAppend(IHyracksTaskContext ctx) throws HyracksDataException {
-        frame = new VSizeFrame(ctx);
-        appender = new FrameTupleAppender(frame);
-        tAccess = new FrameTupleAccessor(inputRecordDesc);
-    }
-
-    protected final void initAccessAppendRef(IHyracksTaskContext ctx) throws HyracksDataException {
-        initAccessAppend(ctx);
-        tRef = new FrameTupleReference();
-    }
-
-    @Override
-    public void close() throws HyracksDataException {
-        flushIfNotFailed();
-        writer.close();
-    }
-
-    protected void flushAndReset() throws HyracksDataException {
-        if (appender.getTupleCount() > 0) {
-            appender.flush(writer, true);
-        }
-    }
-
-    protected void flushIfNotFailed() throws HyracksDataException {
-        if (!failed) {
-            flushAndReset();
-        }
-    }
-
-    protected IFrameTupleAppender getTupleAppender() {
-        return (FrameTupleAppender) appender;
-    }
-
-    protected void appendToFrameFromTupleBuilder(ArrayTupleBuilder tb) throws HyracksDataException {
-        appendToFrameFromTupleBuilder(tb, false);
-    }
-
-    protected void appendToFrameFromTupleBuilder(ArrayTupleBuilder tb, boolean flushFrame) throws HyracksDataException {
-        FrameUtils.appendToWriter(writer, getTupleAppender(), tb.getFieldEndOffsets(), tb.getByteArray(), 0,
-                tb.getSize());
-        if (flushFrame) {
-            flushAndReset();
-        }
-    }
-
-    protected void appendProjectionToFrame(int tIndex, int[] projectionList) throws HyracksDataException {
-        appendProjectionToFrame(tIndex, projectionList, false);
-    }
-
-    protected void appendProjectionToFrame(int tIndex, int[] projectionList, boolean flushFrame)
-            throws HyracksDataException {
-        FrameUtils.appendProjectionToWriter(writer, getTupleAppender(), tAccess, tIndex, projectionList);
-        if (flushFrame) {
-            flushAndReset();
-        }
-    }
-
-    protected void appendTupleToFrame(int tIndex) throws HyracksDataException {
-        FrameUtils.appendToWriter(writer, getTupleAppender(), tAccess, tIndex);
-    }
-
-    protected void appendConcat(IFrameTupleAccessor accessor0, int tIndex0, IFrameTupleAccessor accessor1, int tIndex1)
-            throws HyracksDataException {
-        FrameUtils.appendConcatToWriter(writer, getTupleAppender(), accessor0, tIndex0, accessor1, tIndex1);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
deleted file mode 100644
index 0d3febf..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public abstract class AbstractOneInputOneOutputPushRuntime extends AbstractOneInputPushRuntime {
-
-    protected RecordDescriptor inputRecordDesc;
-
-    @Override
-    public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
-        this.inputRecordDesc = recordDescriptor;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
deleted file mode 100644
index 1d31916..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public abstract class AbstractOneInputOneOutputRuntimeFactory implements IPushRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    protected final int[] projectionList;
-
-    public AbstractOneInputOneOutputRuntimeFactory(int[] projectionList) {
-        this.projectionList = projectionList;
-    }
-
-    @Override
-    public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException, HyracksDataException {
-        return createOneOutputPushRuntime(ctx);
-    }
-
-    public abstract AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(IHyracksTaskContext ctx)
-            throws AlgebricksException, HyracksDataException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java
deleted file mode 100644
index 47ebac1..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputPushRuntime.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public abstract class AbstractOneInputPushRuntime implements IPushRuntime {
-    protected IFrameWriter writer;
-    protected RecordDescriptor outputRecordDesc;
-    protected boolean failed;
-
-    @Override
-    public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
-        this.writer = writer;
-        this.outputRecordDesc = recordDesc;
-    }
-
-    @Override
-    public void fail() throws HyracksDataException {
-        failed = true;
-        writer.fail();
-    }
-}

[20/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java
deleted file mode 100644
index 37460ad..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriter.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-
-public interface IAWriter {
-
-    public void init() throws AlgebricksException;
-
-    public void printTuple(IFrameTupleAccessor tAccess, int tIdx) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java
deleted file mode 100644
index bf7ba87..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IAWriterFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import java.io.PrintStream;
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public interface IAWriterFactory extends Serializable {
-    public IAWriter createWriter(int[] fields, PrintStream ps, IPrinterFactory[] printerFactories,
-            RecordDescriptor inputRecordDescriptor);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java
deleted file mode 100644
index e21a2fa..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspector.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-public interface IBinaryBooleanInspector {
-    public boolean getBooleanValue(byte[] bytes, int offset, int length);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
deleted file mode 100644
index d327cd4..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface IBinaryBooleanInspectorFactory extends Serializable {
-    public IBinaryBooleanInspector createBinaryBooleanInspector(IHyracksTaskContext ctx);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
deleted file mode 100644
index ecdccef..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-
-public interface IBinaryComparatorFactoryProvider {
-    public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean ascending)
-            throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
deleted file mode 100644
index 170ef3a..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-
-public interface IBinaryHashFunctionFactoryProvider {
-    public IBinaryHashFunctionFactory getBinaryHashFunctionFactory(Object type) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java
deleted file mode 100644
index 1d4a77c..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
-
-public interface IBinaryHashFunctionFamilyProvider {
-
-	public IBinaryHashFunctionFamily getBinaryHashFunctionFamily(Object type)
-			throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java
deleted file mode 100644
index 3707912..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspector.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface IBinaryIntegerInspector {
-    public int getIntegerValue(byte[] bytes, int offset, int length) throws HyracksDataException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
deleted file mode 100644
index 00e418b..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface IBinaryIntegerInspectorFactory extends Serializable {
-    public IBinaryIntegerInspector createBinaryIntegerInspector(IHyracksTaskContext ctx);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java
deleted file mode 100644
index fdc5114..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
-
-public interface ILinearizeComparatorFactoryProvider {
-    public ILinearizeComparatorFactory getLinearizeComparatorFactory(Object type, boolean ascending, int dimension)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
deleted file mode 100644
index b008b8f..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-
-public interface INormalizedKeyComputerFactoryProvider {
-    public INormalizedKeyComputerFactory getNormalizedKeyComputerFactory(Object type, boolean ascending);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java
deleted file mode 100644
index 2aa906a..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinter.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import java.io.PrintStream;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public interface IPrinter {
-    public void init() throws AlgebricksException;
-
-    public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java
deleted file mode 100644
index 27cb7eb..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactory.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import java.io.Serializable;
-
-public interface IPrinterFactory extends Serializable {
-    public IPrinter createPrinter();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java
deleted file mode 100644
index 4c5ca09..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IPrinterFactoryProvider.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public interface IPrinterFactoryProvider {
-    public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IResultSerializerFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IResultSerializerFactoryProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IResultSerializerFactoryProvider.java
deleted file mode 100644
index 42d630e..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/IResultSerializerFactoryProvider.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IResultSerializerFactory;
-
-public interface IResultSerializerFactoryProvider extends Serializable {
-    /**
-     * Returns a result serializer factory
-     * 
-     * @param fields
-     *            - A position of the fields in the order it should be written in the output.
-     * @param printerFactories
-     *            - A printer factory array to print the tuple containing different fields.
-     * @param writerFactory
-     *            - A writer factory to write the serialized data to the print stream.
-     * @param inputRecordDesc
-     *            - The record descriptor describing the input frame to be serialized.
-     * @return A new instance of result serialized appender.
-     */
-    public IResultSerializerFactory getAqlResultSerializerFactoryProvider(int[] fields,
-            IPrinterFactory[] printerFactories, IAWriterFactory writerFactory);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java
deleted file mode 100644
index c53fb9c..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ISerializerDeserializerProvider.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-
-public interface ISerializerDeserializerProvider {
-    @SuppressWarnings("unchecked")
-    public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java
deleted file mode 100644
index ba8965d..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/ITypeTraitProvider.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data;
-
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-
-public interface ITypeTraitProvider {
-    public ITypeTraits getTypeTrait(Object type);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
deleted file mode 100644
index a4f8239..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data.impl;
-
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public class BinaryBooleanInspectorImpl implements IBinaryBooleanInspector {
-    public static final IBinaryBooleanInspectorFactory FACTORY = new IBinaryBooleanInspectorFactory() {
-        private static final long serialVersionUID = 1L;
-
-        @Override
-        public IBinaryBooleanInspector createBinaryBooleanInspector(IHyracksTaskContext ctx) {
-            return new BinaryBooleanInspectorImpl();
-        }
-    };
-
-    private BinaryBooleanInspectorImpl() {
-    }
-
-    @Override
-    public boolean getBooleanValue(byte[] bytes, int offset, int length) {
-        return bytes[offset] == 1;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
deleted file mode 100644
index 691d025..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data.impl;
-
-import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-
-public class BinaryIntegerInspectorImpl implements IBinaryIntegerInspector {
-    public static final IBinaryIntegerInspectorFactory FACTORY = new IBinaryIntegerInspectorFactory() {
-        private static final long serialVersionUID = 1L;
-
-        @Override
-        public IBinaryIntegerInspector createBinaryIntegerInspector(IHyracksTaskContext ctx) {
-            return new BinaryIntegerInspectorImpl();
-        }
-    };
-
-    private BinaryIntegerInspectorImpl() {
-    }
-
-    @Override
-    public int getIntegerValue(byte[] bytes, int offset, int length) {
-        return IntegerPointable.getInteger(bytes, offset);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
deleted file mode 100644
index d41f619..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data.impl;
-
-import java.io.IOException;
-import java.io.PrintStream;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-
-public class IntegerPrinterFactory implements IPrinterFactory {
-
-    private static final long serialVersionUID = 1L;
-    public static final IntegerPrinterFactory INSTANCE = new IntegerPrinterFactory();
-
-    private IntegerPrinterFactory() {
-    }
-
-    @Override
-    public IPrinter createPrinter() {
-        return new IPrinter() {
-
-            @Override
-            public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
-                int d = IntegerPointable.getInteger(b, s);
-                try {
-                    WriteValueTools.writeInt(d, ps);
-                } catch (IOException e) {
-                    throw new AlgebricksException(e);
-                }
-            }
-
-            @Override
-            public void init() throws AlgebricksException {
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
deleted file mode 100644
index 442a724..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data.impl;
-
-import java.io.DataOutput;
-
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class NoopNullWriterFactory implements INullWriterFactory {
-
-    private static final long serialVersionUID = 1L;
-    public static final NoopNullWriterFactory INSTANCE = new NoopNullWriterFactory();
-
-    private NoopNullWriterFactory() {
-    }
-
-    @Override
-    public INullWriter createNullWriter() {
-
-        return new INullWriter() {
-
-            @Override
-            public void writeNull(DataOutput out) throws HyracksDataException {
-                // do nothing
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
deleted file mode 100644
index 3b802b3..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data.impl;
-
-import java.io.PrintStream;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-
-public class UTF8StringPrinterFactory implements IPrinterFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    public static final UTF8StringPrinterFactory INSTANCE = new UTF8StringPrinterFactory();
-
-    private UTF8StringPrinterFactory() {
-    }
-
-    @Override
-    public IPrinter createPrinter() {
-        return new IPrinter() {
-
-            @Override
-            public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
-                int strlen = UTF8StringPointable.getUTFLength(b, s);
-                int pos = s + 2;
-                int maxPos = pos + strlen;
-                ps.print("\"");
-                while (pos < maxPos) {
-                    char c = UTF8StringPointable.charAt(b, pos);
-                    switch (c) {
-                        case '\\':
-                        case '"':
-                            ps.print('\\');
-                            break;
-                    }
-                    ps.print(c);
-                    pos += UTF8StringPointable.charSize(b, pos);
-                }
-                ps.print("\"");
-            }
-
-            @Override
-            public void init() throws AlgebricksException {
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java b/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java
deleted file mode 100644
index 3a1203b..0000000
--- a/algebricks/algebricks-data/src/main/java/edu/uci/ics/hyracks/algebricks/data/utils/WriteValueTools.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.data.utils;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-
-public final class WriteValueTools {
-
-    private final static int[] INT_INTERVALS = { 9, 99, 999, 9999, 99999, 999999, 9999999, 99999999, 999999999,
-            Integer.MAX_VALUE };
-    private final static int[] INT_DIVIDERS = { 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 };
-    private final static int[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
-
-    public static void writeInt(int i, OutputStream os) throws IOException {
-        if (i < 0) {
-            if (i == Integer.MIN_VALUE) {
-                os.write("-2147483648".getBytes());
-                return;
-            }
-            os.write('-');
-            i = -i;
-        }
-        int k = 0;
-        for (; k < INT_INTERVALS.length; k++) {
-            if (i <= INT_INTERVALS[k]) {
-                break;
-            }
-        }
-        while (k > 0) {
-            int q = i / INT_DIVIDERS[k - 1];
-            os.write(DIGITS[q % 10]);
-            k--;
-        }
-        // now, print the units
-        os.write(DIGITS[i % 10]);
-    }
-
-    public static void writeLong(long d, OutputStream os) throws IOException {
-        // now, print the units
-        if (d < 0) {
-            if (d == Long.MIN_VALUE) {
-                os.write("-9223372036854775808".getBytes());
-                return;
-            }
-            os.write('-');
-            d = -d;
-        }
-        long divisor = 1000000000000000000L;
-        while (divisor > d) {
-            divisor = divisor / 10;
-        }
-        while (divisor > 1) {
-            os.write(DIGITS[(int) ((d / divisor) % 10)]);
-            divisor = divisor / 10;
-        }
-        os.write(DIGITS[(int) (d % 10)]);
-    }
-
-    public static void writeUTF8String(byte[] b, int s, int l, OutputStream os) throws IOException {
-        int stringLength = UTF8StringPointable.getUTFLength(b, s);
-        int position = s + 2;
-        int maxPosition = position + stringLength;
-        os.write('\"');
-        while (position < maxPosition) {
-            char c = UTF8StringPointable.charAt(b, position);
-            switch (c) {
-            // escape
-                case '\\':
-                case '"':
-                    os.write('\\');
-                    break;
-            }
-            int sz = UTF8StringPointable.charSize(b, position);
-            while (sz > 0) {
-                os.write(b[position]);
-                position++;
-                sz--;
-            }
-        }
-        os.write('\"');
-    }
-
-    public static void writeUTF8StringNoQuotes(byte[] b, int s, int l, OutputStream os) throws IOException {
-        int stringLength = UTF8StringPointable.getUTFLength(b, s);
-        int position = s + 2;
-        int maxPosition = position + stringLength;
-        while (position < maxPosition) {
-            char c = UTF8StringPointable.charAt(b, position);
-            switch (c) {
-            // escape
-                case '\\':
-                case '"':
-                    os.write('\\');
-                    break;
-            }
-            int sz = UTF8StringPointable.charSize(b, position);
-            while (sz > 0) {
-                os.write(b[position]);
-                position++;
-                sz--;
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriter.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriter.java
new file mode 100644
index 0000000..37460ad
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriter.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+
+public interface IAWriter {
+
+    public void init() throws AlgebricksException;
+
+    public void printTuple(IFrameTupleAccessor tAccess, int tIdx) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriterFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriterFactory.java
new file mode 100644
index 0000000..bf7ba87
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IAWriterFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import java.io.PrintStream;
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public interface IAWriterFactory extends Serializable {
+    public IAWriter createWriter(int[] fields, PrintStream ps, IPrinterFactory[] printerFactories,
+            RecordDescriptor inputRecordDescriptor);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspector.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspector.java
new file mode 100644
index 0000000..e21a2fa
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspector.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+public interface IBinaryBooleanInspector {
+    public boolean getBooleanValue(byte[] bytes, int offset, int length);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
new file mode 100644
index 0000000..d327cd4
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryBooleanInspectorFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IBinaryBooleanInspectorFactory extends Serializable {
+    public IBinaryBooleanInspector createBinaryBooleanInspector(IHyracksTaskContext ctx);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
new file mode 100644
index 0000000..ecdccef
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryComparatorFactoryProvider.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
+public interface IBinaryComparatorFactoryProvider {
+    public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, boolean ascending)
+            throws AlgebricksException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
new file mode 100644
index 0000000..170ef3a
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFactoryProvider.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+
+public interface IBinaryHashFunctionFactoryProvider {
+    public IBinaryHashFunctionFactory getBinaryHashFunctionFactory(Object type) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java
new file mode 100644
index 0000000..1d4a77c
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryHashFunctionFamilyProvider.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
+
+public interface IBinaryHashFunctionFamilyProvider {
+
+	public IBinaryHashFunctionFamily getBinaryHashFunctionFamily(Object type)
+			throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspector.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspector.java
new file mode 100644
index 0000000..3707912
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspector.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IBinaryIntegerInspector {
+    public int getIntegerValue(byte[] bytes, int offset, int length) throws HyracksDataException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
new file mode 100644
index 0000000..00e418b
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IBinaryIntegerInspectorFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IBinaryIntegerInspectorFactory extends Serializable {
+    public IBinaryIntegerInspector createBinaryIntegerInspector(IHyracksTaskContext ctx);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java
new file mode 100644
index 0000000..fdc5114
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ILinearizeComparatorFactoryProvider.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
+
+public interface ILinearizeComparatorFactoryProvider {
+    public ILinearizeComparatorFactory getLinearizeComparatorFactory(Object type, boolean ascending, int dimension)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
new file mode 100644
index 0000000..b008b8f
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/INormalizedKeyComputerFactoryProvider.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+
+public interface INormalizedKeyComputerFactoryProvider {
+    public INormalizedKeyComputerFactory getNormalizedKeyComputerFactory(Object type, boolean ascending);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinter.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinter.java
new file mode 100644
index 0000000..2aa906a
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinter.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public interface IPrinter {
+    public void init() throws AlgebricksException;
+
+    public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactory.java
new file mode 100644
index 0000000..27cb7eb
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import java.io.Serializable;
+
+public interface IPrinterFactory extends Serializable {
+    public IPrinter createPrinter();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactoryProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactoryProvider.java
new file mode 100644
index 0000000..4c5ca09
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IPrinterFactoryProvider.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public interface IPrinterFactoryProvider {
+    public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IResultSerializerFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IResultSerializerFactoryProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IResultSerializerFactoryProvider.java
new file mode 100644
index 0000000..42d630e
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/IResultSerializerFactoryProvider.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IResultSerializerFactory;
+
+public interface IResultSerializerFactoryProvider extends Serializable {
+    /**
+     * Returns a result serializer factory
+     * 
+     * @param fields
+     *            - A position of the fields in the order it should be written in the output.
+     * @param printerFactories
+     *            - A printer factory array to print the tuple containing different fields.
+     * @param writerFactory
+     *            - A writer factory to write the serialized data to the print stream.
+     * @param inputRecordDesc
+     *            - The record descriptor describing the input frame to be serialized.
+     * @return A new instance of result serialized appender.
+     */
+    public IResultSerializerFactory getAqlResultSerializerFactoryProvider(int[] fields,
+            IPrinterFactory[] printerFactories, IAWriterFactory writerFactory);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java
new file mode 100644
index 0000000..c53fb9c
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ISerializerDeserializerProvider.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+
+public interface ISerializerDeserializerProvider {
+    @SuppressWarnings("unchecked")
+    public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ITypeTraitProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ITypeTraitProvider.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ITypeTraitProvider.java
new file mode 100644
index 0000000..ba8965d
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/ITypeTraitProvider.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+
+public interface ITypeTraitProvider {
+    public ITypeTraits getTypeTrait(Object type);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
new file mode 100644
index 0000000..a4f8239
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryBooleanInspectorImpl.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.impl;
+
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public class BinaryBooleanInspectorImpl implements IBinaryBooleanInspector {
+    public static final IBinaryBooleanInspectorFactory FACTORY = new IBinaryBooleanInspectorFactory() {
+        private static final long serialVersionUID = 1L;
+
+        @Override
+        public IBinaryBooleanInspector createBinaryBooleanInspector(IHyracksTaskContext ctx) {
+            return new BinaryBooleanInspectorImpl();
+        }
+    };
+
+    private BinaryBooleanInspectorImpl() {
+    }
+
+    @Override
+    public boolean getBooleanValue(byte[] bytes, int offset, int length) {
+        return bytes[offset] == 1;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
new file mode 100644
index 0000000..691d025
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/BinaryIntegerInspectorImpl.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.impl;
+
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+
+public class BinaryIntegerInspectorImpl implements IBinaryIntegerInspector {
+    public static final IBinaryIntegerInspectorFactory FACTORY = new IBinaryIntegerInspectorFactory() {
+        private static final long serialVersionUID = 1L;
+
+        @Override
+        public IBinaryIntegerInspector createBinaryIntegerInspector(IHyracksTaskContext ctx) {
+            return new BinaryIntegerInspectorImpl();
+        }
+    };
+
+    private BinaryIntegerInspectorImpl() {
+    }
+
+    @Override
+    public int getIntegerValue(byte[] bytes, int offset, int length) {
+        return IntegerPointable.getInteger(bytes, offset);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/IntegerPrinterFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
new file mode 100644
index 0000000..d41f619
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/IntegerPrinterFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.impl;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+
+public class IntegerPrinterFactory implements IPrinterFactory {
+
+    private static final long serialVersionUID = 1L;
+    public static final IntegerPrinterFactory INSTANCE = new IntegerPrinterFactory();
+
+    private IntegerPrinterFactory() {
+    }
+
+    @Override
+    public IPrinter createPrinter() {
+        return new IPrinter() {
+
+            @Override
+            public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+                int d = IntegerPointable.getInteger(b, s);
+                try {
+                    WriteValueTools.writeInt(d, ps);
+                } catch (IOException e) {
+                    throw new AlgebricksException(e);
+                }
+            }
+
+            @Override
+            public void init() throws AlgebricksException {
+            }
+        };
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/NoopNullWriterFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
new file mode 100644
index 0000000..442a724
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/NoopNullWriterFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.impl;
+
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class NoopNullWriterFactory implements INullWriterFactory {
+
+    private static final long serialVersionUID = 1L;
+    public static final NoopNullWriterFactory INSTANCE = new NoopNullWriterFactory();
+
+    private NoopNullWriterFactory() {
+    }
+
+    @Override
+    public INullWriter createNullWriter() {
+
+        return new INullWriter() {
+
+            @Override
+            public void writeNull(DataOutput out) throws HyracksDataException {
+                // do nothing
+            }
+        };
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
new file mode 100644
index 0000000..3b802b3
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/impl/UTF8StringPrinterFactory.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.impl;
+
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+
+public class UTF8StringPrinterFactory implements IPrinterFactory {
+
+    private static final long serialVersionUID = 1L;
+
+    public static final UTF8StringPrinterFactory INSTANCE = new UTF8StringPrinterFactory();
+
+    private UTF8StringPrinterFactory() {
+    }
+
+    @Override
+    public IPrinter createPrinter() {
+        return new IPrinter() {
+
+            @Override
+            public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+                int strlen = UTF8StringPointable.getUTFLength(b, s);
+                int pos = s + 2;
+                int maxPos = pos + strlen;
+                ps.print("\"");
+                while (pos < maxPos) {
+                    char c = UTF8StringPointable.charAt(b, pos);
+                    switch (c) {
+                        case '\\':
+                        case '"':
+                            ps.print('\\');
+                            break;
+                    }
+                    ps.print(c);
+                    pos += UTF8StringPointable.charSize(b, pos);
+                }
+                ps.print("\"");
+            }
+
+            @Override
+            public void init() throws AlgebricksException {
+            }
+        };
+    }
+
+}

[34/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
new file mode 100644
index 0000000..1792641
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
@@ -0,0 +1,355 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+
+public abstract class AbstractFunctionCallExpression extends AbstractLogicalExpression {
+
+    public enum FunctionKind {
+        SCALAR,
+        STATEFUL,
+        AGGREGATE,
+        UNNEST
+    }
+
+    protected IFunctionInfo finfo;
+    final private List<Mutable<ILogicalExpression>> arguments;
+    private Object[] opaqueParameters;
+    private final FunctionKind kind;
+    private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
+
+    public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
+            List<Mutable<ILogicalExpression>> arguments) {
+        this.kind = kind;
+        this.finfo = finfo;
+        this.arguments = arguments;
+    }
+
+    public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo) {
+        this.kind = kind;
+        this.finfo = finfo;
+        this.arguments = new ArrayList<Mutable<ILogicalExpression>>();
+    }
+
+    public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
+            Mutable<ILogicalExpression>... expressions) {
+        this(kind, finfo);
+        for (Mutable<ILogicalExpression> e : expressions) {
+            this.arguments.add(e);
+        }
+    }
+
+    public void setOpaqueParameters(Object[] opaqueParameters) {
+        this.opaqueParameters = opaqueParameters;
+    }
+
+    public Object[] getOpaqueParameters() {
+        return opaqueParameters;
+    }
+
+    public FunctionKind getKind() {
+        return kind;
+    }
+
+    protected List<Mutable<ILogicalExpression>> cloneArguments() {
+        List<Mutable<ILogicalExpression>> clonedArgs = new ArrayList<Mutable<ILogicalExpression>>(arguments.size());
+        for (Mutable<ILogicalExpression> e : arguments) {
+            ILogicalExpression e2 = ((AbstractLogicalExpression) e.getValue()).cloneExpression();
+            clonedArgs.add(new MutableObject<ILogicalExpression>(e2));
+        }
+        return clonedArgs;
+    }
+
+    public FunctionIdentifier getFunctionIdentifier() {
+        return finfo.getFunctionIdentifier();
+    }
+
+    public IFunctionInfo getFunctionInfo() {
+        return finfo;
+    }
+
+    public void setFunctionInfo(IFunctionInfo finfo) {
+        this.finfo = finfo;
+    }
+
+    public List<Mutable<ILogicalExpression>> getArguments() {
+        return arguments;
+    }
+
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("function-call: " + finfo.getFunctionIdentifier() + ", Args:[");
+        // + arguments;
+        boolean first = true;
+        for (Mutable<ILogicalExpression> ref : arguments) {
+            if (first) {
+                first = false;
+            } else {
+                sb.append(", ");
+            }
+            sb.append(ref.getValue());
+        }
+        sb.append("]");
+        return sb.toString();
+    }
+
+    @Override
+    public LogicalExpressionTag getExpressionTag() {
+        return LogicalExpressionTag.FUNCTION_CALL;
+    }
+
+    @Override
+    public void getUsedVariables(Collection<LogicalVariable> vars) {
+        for (Mutable<ILogicalExpression> arg : arguments) {
+            arg.getValue().getUsedVariables(vars);
+        }
+    }
+
+    @Override
+    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+        for (Mutable<ILogicalExpression> arg : arguments) {
+            arg.getValue().substituteVar(v1, v2);
+        }
+    }
+
+    @Override
+    public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
+            Map<LogicalVariable, EquivalenceClass> equivClasses) {
+        FunctionIdentifier funId = getFunctionIdentifier();
+        if (funId.equals(AlgebricksBuiltinFunctions.AND)) {
+            for (Mutable<ILogicalExpression> a : arguments) {
+                a.getValue().getConstraintsAndEquivClasses(fds, equivClasses);
+            }
+        } else if (funId.equals(AlgebricksBuiltinFunctions.EQ)) {
+            ILogicalExpression opLeft = arguments.get(0).getValue();
+            ILogicalExpression opRight = arguments.get(1).getValue();
+            if (opLeft.getExpressionTag() == LogicalExpressionTag.CONSTANT
+                    && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                ConstantExpression op1 = (ConstantExpression) opLeft;
+                VariableReferenceExpression op2 = (VariableReferenceExpression) opRight;
+                getFDsAndEquivClassesForEqWithConstant(op1, op2, fds, equivClasses);
+            } else if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
+                    && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression op1 = (VariableReferenceExpression) opLeft;
+                VariableReferenceExpression op2 = (VariableReferenceExpression) opRight;
+                getFDsAndEquivClassesForColumnEq(op1, op2, fds, equivClasses);
+            }
+        }
+    }
+
+    @Override
+    public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars) {
+        FunctionIdentifier funId = getFunctionIdentifier();
+        if (funId.equals(AlgebricksBuiltinFunctions.AND)) {
+            for (Mutable<ILogicalExpression> a : arguments) {
+                a.getValue().getConstraintsForOuterJoin(fds, outerVars);
+            }
+        } else if (funId.equals(AlgebricksBuiltinFunctions.EQ)) {
+            ILogicalExpression opLeft = arguments.get(0).getValue();
+            ILogicalExpression opRight = arguments.get(1).getValue();
+            if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
+                    && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
+                LogicalVariable var2 = ((VariableReferenceExpression) opRight).getVariableReference();
+                if (outerVars.contains(var1)) {
+                    addFD(fds, var1, var2);
+                }
+                if (outerVars.contains(var2)) {
+                    addFD(fds, var2, var1);
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof AbstractFunctionCallExpression)) {
+            return false;
+        } else {
+            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) obj;
+            boolean equal = getFunctionIdentifier().equals(fce.getFunctionIdentifier());
+            if (!equal)
+                return false;
+            for (int i = 0; i < arguments.size(); i++) {
+                ILogicalExpression argument = arguments.get(i).getValue();
+                ILogicalExpression fceArgument = fce.getArguments().get(i).getValue();
+                if (!argument.equals(fceArgument))
+                    return false;
+            }
+            if (opaqueParameters != null) {
+                if (opaqueParameters.length != fce.opaqueParameters.length)
+                    return false;
+                for (int i = 0; i < opaqueParameters.length; i++) {
+                    Object opaqueParameter = opaqueParameters[i];
+                    Object fceOpaqueParameter = fce.opaqueParameters[i];
+                    if (!opaqueParameter.equals(fceOpaqueParameter))
+                        return false;
+                }
+            }
+            return true;
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        int h = finfo.hashCode();
+        for (Mutable<ILogicalExpression> e : arguments) {
+            h = h * 41 + e.getValue().hashCode();
+        }
+        if (opaqueParameters != null) {
+            for (int i = 0; i < opaqueParameters.length; i++) {
+                h = h * 31 + opaqueParameters[i].hashCode();
+            }
+        }
+        return h;
+    }
+
+    @Override
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
+        if (!getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND) || arguments.size() <= 1) {
+            return false;
+        } else {
+            conjs.addAll(arguments);
+            return true;
+        }
+    }
+
+    public Map<Object, IExpressionAnnotation> getAnnotations() {
+        return annotationMap;
+    }
+
+    protected Map<Object, IExpressionAnnotation> cloneAnnotations() {
+        Map<Object, IExpressionAnnotation> m = new HashMap<Object, IExpressionAnnotation>();
+        for (Object k : annotationMap.keySet()) {
+            IExpressionAnnotation annot2 = annotationMap.get(k).copy();
+            m.put(k, annot2);
+        }
+        return m;
+    }
+
+    private final static void addFD(Collection<FunctionalDependency> fds, LogicalVariable var1, LogicalVariable var2) {
+        LinkedList<LogicalVariable> set1 = new LinkedList<LogicalVariable>();
+        set1.add(var1);
+        LinkedList<LogicalVariable> set2 = new LinkedList<LogicalVariable>();
+        set2.add(var2);
+        FunctionalDependency fd1 = new FunctionalDependency(set1, set2);
+        fds.add(fd1);
+    }
+
+    private final static void getFDsAndEquivClassesForEqWithConstant(ConstantExpression c,
+            VariableReferenceExpression v, Collection<FunctionalDependency> fds,
+            Map<LogicalVariable, EquivalenceClass> equivClasses) {
+        LogicalVariable var = v.getVariableReference();
+        LinkedList<LogicalVariable> head = new LinkedList<LogicalVariable>();
+        // empty set in the head
+        LinkedList<LogicalVariable> tail = new LinkedList<LogicalVariable>();
+        tail.add(var);
+        FunctionalDependency fd = new FunctionalDependency(head, tail);
+        fds.add(fd);
+
+        EquivalenceClass ec = equivClasses.get(var);
+        if (ec == null) {
+            LinkedList<LogicalVariable> members = new LinkedList<LogicalVariable>();
+            members.add(var);
+            EquivalenceClass eclass = new EquivalenceClass(members, c);
+            equivClasses.put(var, eclass);
+        } else {
+            if (ec.representativeIsConst()) {
+                ILogicalExpression c1 = ec.getConstRepresentative();
+                if (!c1.equals(c)) {
+                    // here I could also rewrite to FALSE
+                    return;
+                }
+            }
+            ec.setConstRepresentative(c);
+        }
+    }
+
+    /*
+     * Obs.: mgmt. of equiv. classes should use a more efficient data
+     * structure,if we are to implem. cost-bazed optim.
+     */
+    private final static void getFDsAndEquivClassesForColumnEq(VariableReferenceExpression v1,
+            VariableReferenceExpression v2, Collection<FunctionalDependency> fds,
+            Map<LogicalVariable, EquivalenceClass> equivClasses) {
+        LogicalVariable var1 = v1.getVariableReference();
+        LogicalVariable var2 = v2.getVariableReference();
+        LinkedList<LogicalVariable> set1 = new LinkedList<LogicalVariable>();
+        set1.add(var1);
+        LinkedList<LogicalVariable> set2 = new LinkedList<LogicalVariable>();
+        set2.add(var2);
+        FunctionalDependency fd1 = new FunctionalDependency(set1, set2);
+        FunctionalDependency fd2 = new FunctionalDependency(set2, set1);
+        fds.add(fd1);
+        fds.add(fd2);
+
+        EquivalenceClass ec1 = equivClasses.get(var1);
+        EquivalenceClass ec2 = equivClasses.get(var2);
+        if (ec1 == null && ec2 == null) {
+            LinkedList<LogicalVariable> members = new LinkedList<LogicalVariable>();
+            members.add(var1);
+            members.add(var2);
+            EquivalenceClass ec = new EquivalenceClass(members, var1);
+            equivClasses.put(var1, ec);
+            equivClasses.put(var2, ec);
+        } else if (ec1 == null && ec2 != null) {
+            ec2.addMember(var1);
+            equivClasses.put(var1, ec2);
+        } else if (ec2 == null && ec1 != null) {
+            ec1.addMember(var2);
+            equivClasses.put(var2, ec1);
+        } else {
+            ec1.merge(ec2);
+            for (LogicalVariable w : equivClasses.keySet()) {
+                if (ec2.getMembers().contains(w)) {
+                    equivClasses.put(w, ec1);
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean isFunctional() {
+        if (!finfo.isFunctional()) {
+            return false;
+        }
+
+        for (Mutable<ILogicalExpression> e : arguments) {
+            if (!e.getValue().isFunctional()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
new file mode 100644
index 0000000..2718076
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.Collection;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+
+public abstract class AbstractLogicalExpression implements ILogicalExpression {
+
+    @Override
+    public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
+            Map<LogicalVariable, EquivalenceClass> equivClasses) {
+        // do nothing
+    }
+
+    @Override
+    public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds,
+            Collection<LogicalVariable> innerSideVars) {
+        // do nothing
+    }
+
+    @Override
+    public boolean isFunctional() {
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
new file mode 100644
index 0000000..154618b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+/**
+ * 
+ * An aggregate function may be executed in a "two step" mode. First the
+ * "step-one" aggregates are run and then the results are passed to the
+ * "step-two" aggregators. The convention is the following:
+ * 
+ * 1. The step-one aggregate must be able to accept the same arguments as the
+ * original aggregate function call.
+ * 
+ * 2. The step-two aggregate must be a unary function that accepts as input the
+ * output of the step-one aggregate.
+ * 
+ */
+
+public class AggregateFunctionCallExpression extends AbstractFunctionCallExpression {
+
+    private boolean twoStep;
+    private IFunctionInfo stepOneAggregate;
+    private IFunctionInfo stepTwoAggregate;
+
+    public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep) {
+        super(FunctionKind.AGGREGATE, finfo);
+        this.twoStep = isTwoStep;
+    }
+
+    public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
+            List<Mutable<ILogicalExpression>> arguments) {
+        super(FunctionKind.AGGREGATE, finfo, arguments);
+        this.twoStep = isTwoStep;
+    }
+
+    public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
+            Mutable<ILogicalExpression>... expressions) {
+        super(FunctionKind.AGGREGATE, finfo, expressions);
+        this.twoStep = isTwoStep;
+    }
+
+    public boolean isTwoStep() {
+        return twoStep;
+    }
+
+    public void setTwoStep(boolean twoStep) {
+        this.twoStep = twoStep;
+    }
+
+    @Override
+    public AggregateFunctionCallExpression cloneExpression() {
+        cloneAnnotations();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+        AggregateFunctionCallExpression fun = new AggregateFunctionCallExpression(finfo, twoStep, clonedArgs);
+        fun.setStepTwoAggregate(stepTwoAggregate);
+        fun.setStepOneAggregate(stepOneAggregate);
+        return fun;
+    }
+
+    public void setStepOneAggregate(IFunctionInfo stepOneAggregate) {
+        this.stepOneAggregate = stepOneAggregate;
+    }
+
+    public IFunctionInfo getStepOneAggregate() {
+        return stepOneAggregate;
+    }
+
+    public void setStepTwoAggregate(IFunctionInfo stepTwoAggregate) {
+        this.stepTwoAggregate = stepTwoAggregate;
+    }
+
+    public IFunctionInfo getStepTwoAggregate() {
+        return stepTwoAggregate;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitAggregateFunctionCallExpression(this, arg);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
new file mode 100644
index 0000000..a29bd46
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public class BroadcastExpressionAnnotation implements IExpressionAnnotation {
+
+    public static final String BROADCAST_ANNOTATION_KEY = "broadcast";
+
+    public enum BroadcastSide {
+        LEFT,
+        RIGHT
+    };
+
+    private BroadcastSide side;
+
+    @Override
+    public Object getObject() {
+        return side;
+    }
+
+    @Override
+    public void setObject(Object side) {
+        this.side = (BroadcastSide) side;
+    }
+
+    @Override
+    public IExpressionAnnotation copy() {
+        BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
+        bcast.side = side;
+        return bcast;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
new file mode 100644
index 0000000..1a3fe47
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
@@ -0,0 +1,174 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+public final class ConstantExpression extends AbstractLogicalExpression {
+    private IAlgebricksConstantValue value;
+
+    public final static ConstantExpression TRUE = new ConstantExpression(new IAlgebricksConstantValue() {
+
+        @Override
+        public boolean isTrue() {
+            return true;
+        }
+
+        @Override
+        public boolean isNull() {
+            return false;
+        }
+
+        @Override
+        public boolean isFalse() {
+            return false;
+        }
+
+        @Override
+        public String toString() {
+            return "TRUE";
+        }
+    });
+    public final static ConstantExpression FALSE = new ConstantExpression(new IAlgebricksConstantValue() {
+
+        @Override
+        public boolean isTrue() {
+            return false;
+        }
+
+        @Override
+        public boolean isNull() {
+            return false;
+        }
+
+        @Override
+        public boolean isFalse() {
+            return true;
+        }
+
+        @Override
+        public String toString() {
+            return "FALSE";
+        }
+    });
+    public final static ConstantExpression NULL = new ConstantExpression(new IAlgebricksConstantValue() {
+
+        @Override
+        public boolean isTrue() {
+            return false;
+        }
+
+        @Override
+        public boolean isNull() {
+            return true;
+        }
+
+        @Override
+        public boolean isFalse() {
+            return false;
+        }
+
+        @Override
+        public String toString() {
+            return "NULL";
+        }
+    });
+
+    private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
+
+    public ConstantExpression(IAlgebricksConstantValue value) {
+        this.value = value;
+    }
+
+    public IAlgebricksConstantValue getValue() {
+        return value;
+    }
+
+    public void setValue(IAlgebricksConstantValue value) {
+        this.value = value;
+    }
+
+    @Override
+    public LogicalExpressionTag getExpressionTag() {
+        return LogicalExpressionTag.CONSTANT;
+    }
+
+    @Override
+    public String toString() {
+        return value.toString();
+    }
+
+    @Override
+    public void getUsedVariables(Collection<LogicalVariable> vars) {
+        // do nothing
+    }
+
+    @Override
+    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+        // do nothing
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof ConstantExpression)) {
+            return false;
+        } else {
+            return value.equals(((ConstantExpression) obj).getValue());
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return value.hashCode();
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitConstantExpression(this, arg);
+    }
+
+    @Override
+    public AbstractLogicalExpression cloneExpression() {
+        Map<Object, IExpressionAnnotation> m = new HashMap<Object, IExpressionAnnotation>();
+        for (Object k : annotationMap.keySet()) {
+            IExpressionAnnotation annot2 = annotationMap.get(k).copy();
+            m.put(k, annot2);
+        }
+        ConstantExpression c = new ConstantExpression(value);
+        c.annotationMap = m;
+        return c;
+    }
+
+    public Map<Object, IExpressionAnnotation> getAnnotations() {
+        return annotationMap;
+    }
+
+    @Override
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
+        return false;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
new file mode 100644
index 0000000..975014d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public class ExpressionAnnotationNoCopyImpl implements IExpressionAnnotation {
+
+    private Object object;
+
+    @Override
+    public IExpressionAnnotation copy() {
+        return this;
+    }
+
+    @Override
+    public Object getObject() {
+        return object;
+    }
+
+    @Override
+    public void setObject(Object object) {
+        this.object = object;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
new file mode 100644
index 0000000..a12b8f2
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public interface IAlgebricksConstantValue {
+    public boolean isNull();
+
+    public boolean isTrue();
+
+    public boolean isFalse();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
new file mode 100644
index 0000000..fa2a6f4
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public interface IExpressionAnnotation {
+    public Object getObject();
+
+    public void setObject(Object object);
+
+    public IExpressionAnnotation copy();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
new file mode 100644
index 0000000..24befcc
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
+public interface IExpressionEvalSizeComputer {
+    // size in bytes, or -1 if unknown
+    public int getEvalSize(ILogicalExpression expr, IVariableEvalSizeEnvironment env) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
new file mode 100644
index 0000000..f44cd29
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+
+public interface IExpressionRuntimeProvider {
+    public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
+
+    public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException;
+
+    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+            JobGenContext context) throws AlgebricksException;
+
+    public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException;
+
+    public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
new file mode 100644
index 0000000..87c5ade
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface IExpressionTypeComputer {
+    public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
+            IVariableTypeEnvironment env) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
new file mode 100644
index 0000000..83e36b2
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
+
+public interface ILogicalExpressionJobGen {
+
+    public ICopyEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
+
+    public ICopyAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException;
+
+    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+            JobGenContext context) throws AlgebricksException;
+
+    public ICopyRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException;
+
+    public ICopyUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
new file mode 100644
index 0000000..6a6989d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IMergeAggregationExpressionFactory {
+    ILogicalExpression createMergeAggregation(LogicalVariable originalAggVariable, ILogicalExpression expr, IOptimizationContext env)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
new file mode 100644
index 0000000..d3f6d19
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public interface INullableTypeComputer {
+    public Object makeNullableType(Object type) throws AlgebricksException;
+
+    public boolean canBeNull(Object type);
+
+    public Object getNonOptionalType(Object type);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
new file mode 100644
index 0000000..4d9fe38
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface IPartialAggregationTypeComputer {
+    public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env,
+            IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
new file mode 100644
index 0000000..351f4a2
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IVariableEvalSizeEnvironment {
+    // size in bytes or -1 if unknown
+    public int getVariableEvalSize(LogicalVariable var);
+
+    public void setVariableEvalSize(LogicalVariable var, int size);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
new file mode 100644
index 0000000..8bcc575
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IVariableTypeEnvironment {
+    public Object getVarType(LogicalVariable var) throws AlgebricksException;
+
+    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
+            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException;
+
+    public void setVarType(LogicalVariable var, Object type);
+
+    public Object getType(ILogicalExpression expr) throws AlgebricksException;
+
+    public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
new file mode 100644
index 0000000..97a3f9a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public class IndexedNLJoinExpressionAnnotation implements IExpressionAnnotation {
+
+    public static final String INDEXED_NL_JOIN_ANNOTATION_KEY = "indexnl";
+    public static final IndexedNLJoinExpressionAnnotation INSTANCE = new IndexedNLJoinExpressionAnnotation();
+
+    private Object object;
+
+    @Override
+    public Object getObject() {
+        return object;
+    }
+
+    @Override
+    public void setObject(Object object) {
+        this.object = object;
+    }
+
+    @Override
+    public IExpressionAnnotation copy() {
+        IndexedNLJoinExpressionAnnotation clone = new IndexedNLJoinExpressionAnnotation();
+        clone.setObject(object);
+        return clone;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
new file mode 100644
index 0000000..4241146
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
@@ -0,0 +1,214 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunction;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class LogicalExpressionJobGenToExpressionRuntimeProviderAdapter implements IExpressionRuntimeProvider {
+    private final ILogicalExpressionJobGen lejg;
+
+    public LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(ILogicalExpressionJobGen lejg) {
+        this.lejg = lejg;
+    }
+
+    @Override
+    public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
+        ICopyEvaluatorFactory cef = lejg.createEvaluatorFactory(expr, env, inputSchemas, context);
+        return new ScalarEvaluatorFactoryAdapter(cef);
+    }
+
+    @Override
+    public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException {
+        ICopyAggregateFunctionFactory caff = lejg.createAggregateFunctionFactory(expr, env, inputSchemas, context);
+        return new AggregateFunctionFactoryAdapter(caff);
+    }
+
+    @Override
+    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+            JobGenContext context) throws AlgebricksException {
+        return lejg.createSerializableAggregateFunctionFactory(expr, env, inputSchemas, context);
+    }
+
+    @Override
+    public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException {
+        ICopyRunningAggregateFunctionFactory craff = lejg.createRunningAggregateFunctionFactory(expr, env,
+                inputSchemas, context);
+        return new RunningAggregateFunctionFactoryAdapter(craff);
+    }
+
+    @Override
+    public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException {
+        ICopyUnnestingFunctionFactory cuff = lejg.createUnnestingFunctionFactory(expr, env, inputSchemas, context);
+        return new UnnestingFunctionFactoryAdapter(cuff);
+    }
+
+    public static final class ScalarEvaluatorFactoryAdapter implements IScalarEvaluatorFactory {
+        private static final long serialVersionUID = 1L;
+
+        private final ICopyEvaluatorFactory cef;
+
+        public ScalarEvaluatorFactoryAdapter(ICopyEvaluatorFactory cef) {
+            this.cef = cef;
+        }
+
+        @Override
+        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+            final ICopyEvaluator ce = cef.createEvaluator(abvs);
+            return new IScalarEvaluator() {
+                @Override
+                public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+                    abvs.reset();
+                    ce.evaluate(tuple);
+                    result.set(abvs);
+                }
+            };
+        }
+    }
+
+    public static final class AggregateFunctionFactoryAdapter implements IAggregateEvaluatorFactory {
+        private static final long serialVersionUID = 1L;
+
+        private final ICopyAggregateFunctionFactory caff;
+
+        public AggregateFunctionFactoryAdapter(ICopyAggregateFunctionFactory caff) {
+            this.caff = caff;
+        }
+
+        @Override
+        public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+            final ICopyAggregateFunction caf = caff.createAggregateFunction(abvs);
+            return new IAggregateEvaluator() {
+                @Override
+                public void step(IFrameTupleReference tuple) throws AlgebricksException {
+                    caf.step(tuple);
+                }
+
+                @Override
+                public void init() throws AlgebricksException {
+                    abvs.reset();
+                    caf.init();
+                }
+
+                @Override
+                public void finishPartial(IPointable result) throws AlgebricksException {
+                    caf.finishPartial();
+                    result.set(abvs);
+                }
+
+                @Override
+                public void finish(IPointable result) throws AlgebricksException {
+                    caf.finish();
+                    result.set(abvs);
+                }
+
+            };
+        }
+    }
+
+    public static final class RunningAggregateFunctionFactoryAdapter implements IRunningAggregateEvaluatorFactory {
+        private static final long serialVersionUID = 1L;
+
+        private final ICopyRunningAggregateFunctionFactory craff;
+
+        public RunningAggregateFunctionFactoryAdapter(ICopyRunningAggregateFunctionFactory craff) {
+            this.craff = craff;
+        }
+
+        @Override
+        public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException {
+            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+            final ICopyRunningAggregateFunction craf = craff.createRunningAggregateFunction(abvs);
+            return new IRunningAggregateEvaluator() {
+                @Override
+                public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
+                    abvs.reset();
+                    craf.step(tuple);
+                    result.set(abvs);
+                }
+
+                @Override
+                public void init() throws AlgebricksException {
+                    craf.init();
+                }
+            };
+        }
+    }
+
+    public static final class UnnestingFunctionFactoryAdapter implements IUnnestingEvaluatorFactory {
+        private static final long serialVersionUID = 1L;
+
+        private final ICopyUnnestingFunctionFactory cuff;
+
+        public UnnestingFunctionFactoryAdapter(ICopyUnnestingFunctionFactory cuff) {
+            this.cuff = cuff;
+        }
+
+        @Override
+        public IUnnestingEvaluator createUnnestingEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
+            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+            final ICopyUnnestingFunction cuf = cuff.createUnnestingFunction(abvs);
+            return new IUnnestingEvaluator() {
+                @Override
+                public boolean step(IPointable result) throws AlgebricksException {
+                    abvs.reset();
+                    if (cuf.step()) {
+                        result.set(abvs);
+                        return true;
+                    }
+                    return false;
+                }
+
+                @Override
+                public void init(IFrameTupleReference tuple) throws AlgebricksException {
+                    cuf.init(tuple);
+                }
+            };
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
new file mode 100644
index 0000000..81f3dab
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+public class ScalarFunctionCallExpression extends AbstractFunctionCallExpression {
+
+    public ScalarFunctionCallExpression(IFunctionInfo finfo) {
+        super(FunctionKind.SCALAR, finfo);
+    }
+
+    public ScalarFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
+        super(FunctionKind.SCALAR, finfo, arguments);
+    }
+
+    public ScalarFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
+        super(FunctionKind.SCALAR, finfo, expressions);
+    }
+
+    @Override
+    public ScalarFunctionCallExpression cloneExpression() {
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+        ScalarFunctionCallExpression funcExpr = new ScalarFunctionCallExpression(finfo, clonedArgs);
+        funcExpr.getAnnotations().putAll(cloneAnnotations());
+        funcExpr.setOpaqueParameters(this.getOpaqueParameters());
+        return funcExpr;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitScalarFunctionCallExpression(this, arg);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
new file mode 100644
index 0000000..63a5d1a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPropertiesComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+public class StatefulFunctionCallExpression extends AbstractFunctionCallExpression {
+
+    private final IPropertiesComputer propertiesComputer;
+
+    public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer) {
+        super(FunctionKind.STATEFUL, finfo);
+        this.propertiesComputer = propertiesComputer;
+    }
+
+    public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
+            List<Mutable<ILogicalExpression>> arguments) {
+        super(FunctionKind.STATEFUL, finfo, arguments);
+        this.propertiesComputer = propertiesComputer;
+    }
+
+    public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
+            Mutable<ILogicalExpression>... expressions) {
+        super(FunctionKind.STATEFUL, finfo, expressions);
+        this.propertiesComputer = propertiesComputer;
+    }
+
+    @Override
+    public StatefulFunctionCallExpression cloneExpression() {
+        cloneAnnotations();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+        return new StatefulFunctionCallExpression(finfo, propertiesComputer, clonedArgs);
+    }
+
+    // can be null
+    public IPartitioningProperty getRequiredPartitioningProperty() {
+        return propertiesComputer.computePartitioningProperty(this);
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitStatefulFunctionCallExpression(this, arg);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
new file mode 100644
index 0000000..562e8a1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+public class UnnestingFunctionCallExpression extends AbstractFunctionCallExpression {
+
+    private boolean returnsUniqueValues;
+
+    public UnnestingFunctionCallExpression(IFunctionInfo finfo) {
+        super(FunctionKind.UNNEST, finfo);
+    }
+
+    public UnnestingFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
+        super(FunctionKind.UNNEST, finfo, arguments);
+    }
+
+    public UnnestingFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
+        super(FunctionKind.UNNEST, finfo, expressions);
+    }
+
+    @Override
+    public UnnestingFunctionCallExpression cloneExpression() {
+        cloneAnnotations();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+        UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(finfo, clonedArgs);
+        ufce.setReturnsUniqueValues(returnsUniqueValues);
+        ufce.setOpaqueParameters(this.getOpaqueParameters());
+        return ufce;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitUnnestingFunctionCallExpression(this, arg);
+    }
+
+    public void setReturnsUniqueValues(boolean returnsUniqueValues) {
+        this.returnsUniqueValues = returnsUniqueValues;
+    }
+
+    public boolean returnsUniqueValues() {
+        return returnsUniqueValues;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
new file mode 100644
index 0000000..0ab1443
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+public class VariableReferenceExpression extends AbstractLogicalExpression {
+    private int tupleRef;
+    private LogicalVariable variable;
+
+    public VariableReferenceExpression(int tupleRef, LogicalVariable variable) {
+        this.tupleRef = tupleRef;
+        this.variable = variable;
+    }
+
+    public VariableReferenceExpression(LogicalVariable variable) {
+        this(0, variable);
+    }
+
+    public int getTupleRef() {
+        return tupleRef;
+    }
+
+    public void setTupleRef(int tupleRef) {
+        this.tupleRef = tupleRef;
+    }
+
+    public LogicalVariable getVariableReference() {
+        return variable;
+    }
+
+    public void setVariable(LogicalVariable variable) {
+        this.variable = variable;
+    }
+
+    @Override
+    public LogicalExpressionTag getExpressionTag() {
+        return LogicalExpressionTag.VARIABLE;
+    }
+
+    @Override
+    public String toString() {
+        return "%" + tupleRef + "->" + variable.toString();
+    }
+
+    @Override
+    public void getUsedVariables(Collection<LogicalVariable> vars) {
+        // if (!vars.contains(variable)) {
+        vars.add(variable);
+        // }
+    }
+
+    @Override
+    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+        if (variable.equals(v1)) {
+            variable = v2;
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof VariableReferenceExpression)) {
+            return false;
+        } else {
+            return tupleRef == ((VariableReferenceExpression) obj).tupleRef
+                    && variable.equals(((VariableReferenceExpression) obj).getVariableReference());
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return tupleRef + variable.getId();
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitVariableReferenceExpression(this, arg);
+    }
+
+    @Override
+    public AbstractLogicalExpression cloneExpression() {
+        return new VariableReferenceExpression(variable);
+    }
+
+    @Override
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
+        return false;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java
new file mode 100644
index 0000000..b1d6ec6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+import java.io.Serializable;
+
+public abstract class AbstractFunctionInfo implements IFunctionInfo, Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private final boolean isFunctional;
+
+    protected AbstractFunctionInfo(boolean isFunctional) {
+        this.isFunctional = isFunctional;
+    }
+
+    @Override
+    public boolean isFunctional() {
+        return isFunctional;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
new file mode 100644
index 0000000..5d75c5c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AlgebricksBuiltinFunctions {
+    public enum ComparisonKind {
+        EQ,
+        LE,
+        GE,
+        LT,
+        GT,
+        NEQ
+    }
+
+    public static final String ALGEBRICKS_NS = "algebricks";
+
+    // comparisons
+    public final static FunctionIdentifier EQ = new FunctionIdentifier(ALGEBRICKS_NS, "eq", 2);
+    public final static FunctionIdentifier LE = new FunctionIdentifier(ALGEBRICKS_NS, "le", 2);
+    public final static FunctionIdentifier GE = new FunctionIdentifier(ALGEBRICKS_NS, "ge", 2);
+    public final static FunctionIdentifier LT = new FunctionIdentifier(ALGEBRICKS_NS, "lt", 2);
+    public final static FunctionIdentifier GT = new FunctionIdentifier(ALGEBRICKS_NS, "gt", 2);
+    public final static FunctionIdentifier NEQ = new FunctionIdentifier(ALGEBRICKS_NS, "neq", 2);
+
+    // booleans
+    public final static FunctionIdentifier NOT = new FunctionIdentifier(ALGEBRICKS_NS, "not", 1);
+    public final static FunctionIdentifier AND = new FunctionIdentifier(ALGEBRICKS_NS, "and",
+            FunctionIdentifier.VARARGS);
+    public final static FunctionIdentifier OR = new FunctionIdentifier(ALGEBRICKS_NS, "or", FunctionIdentifier.VARARGS);
+
+    // numerics
+    public final static FunctionIdentifier NUMERIC_ADD = new FunctionIdentifier(ALGEBRICKS_NS, "numeric-add", 2);
+
+    // nulls
+    public final static FunctionIdentifier IS_NULL = new FunctionIdentifier(ALGEBRICKS_NS, "is-null", 1);
+
+    private static final Map<FunctionIdentifier, ComparisonKind> comparisonFunctions = new HashMap<FunctionIdentifier, ComparisonKind>();
+    static {
+        comparisonFunctions.put(AlgebricksBuiltinFunctions.EQ, ComparisonKind.EQ);
+        comparisonFunctions.put(AlgebricksBuiltinFunctions.LE, ComparisonKind.LE);
+        comparisonFunctions.put(AlgebricksBuiltinFunctions.GE, ComparisonKind.GE);
+        comparisonFunctions.put(AlgebricksBuiltinFunctions.LT, ComparisonKind.LT);
+        comparisonFunctions.put(AlgebricksBuiltinFunctions.GT, ComparisonKind.GT);
+        comparisonFunctions.put(AlgebricksBuiltinFunctions.NEQ, ComparisonKind.NEQ);
+    }
+
+    public static ComparisonKind getComparisonType(FunctionIdentifier fi) {
+        return comparisonFunctions.get(fi);
+    }
+
+    public static boolean isComparisonFunction(FunctionIdentifier fi) {
+        return comparisonFunctions.get(fi) != null;
+    }
+}

[32/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
new file mode 100644
index 0000000..bdf6f1c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+/**
+ * @author rico
+ */
+public class ExtensionOperator extends AbstractLogicalOperator {
+
+    private IOperatorExtension delegate;
+
+    public ExtensionOperator(IOperatorExtension delegate) {
+        super();
+        if (delegate == null) {
+            throw new IllegalArgumentException("delegate cannot be null!");
+        }
+        this.delegate = delegate;
+        setExecutionMode(delegate.getExecutionMode());
+    }
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+        delegate.setSchema(schema);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+        return delegate.acceptExpressionTransform(transform);
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitExtensionOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return this.delegate.isMap();
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return this.createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.EXTENSION_OPERATOR;
+    }
+
+    public IOperatorExtension getNewInstanceOfDelegateOperator() {
+        return delegate.newInstance();
+    }
+
+    @Override
+    public List<LogicalVariable> getSchema() {
+        return this.schema;
+    }
+
+    @Override
+    public ExecutionMode getExecutionMode() {
+        return delegate.getExecutionMode();
+    }
+
+    @Override
+    public void setExecutionMode(ExecutionMode mode) {
+        delegate.setExecutionMode(mode);
+    }
+
+    @Override
+    public IPhysicalOperator getPhysicalOperator() {
+        return delegate.getPhysicalOperator();
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return this.createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    @Override
+    public String toString() {
+        return delegate.toString();
+    }
+    
+    public IOperatorExtension getDelegate() {
+        return delegate;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
new file mode 100644
index 0000000..4737cfc
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ExternalDataLookupOperator extends AbstractDataSourceOperator {
+
+    private final List<Object> variableTypes;
+    protected final Mutable<ILogicalExpression> expression;
+    private final boolean propagateInput;
+
+    public ExternalDataLookupOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
+            List<Object> variableTypes, boolean propagateInput, IDataSource<?> dataSource) {
+        super(variables, dataSource);
+        this.expression = expression;
+        this.variableTypes = variableTypes;
+        this.propagateInput = propagateInput;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.EXTERNAL_LOOKUP;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = null;
+        if (propagateInput) {
+            env = createPropagatingAllInputsTypeEnvironment(ctx);
+        } else {
+            env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMetadataProvider());
+        }
+        env.setVarType(variables.get(0), variableTypes.get(0));
+        return env;
+    }
+
+    public List<Object> getVariableTypes() {
+        return variableTypes;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitExternalDataLookupOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    public boolean isPropagateInput() {
+        return propagateInput;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (propagateInput) {
+                    ArrayList<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getUsedVariables(ExternalDataLookupOperator.this, usedVariables);
+                    int numOfSources = sources.length;
+                    for (int i = 0; i < numOfSources; i++) {
+                        for (LogicalVariable v : sources[i]) {
+                            if (!usedVariables.contains(v)) {
+                                target.addVariable(v);
+                            }
+                        }
+                    }
+                }
+                target.addVariable(variables.get(0));
+            }
+        };
+    }
+
+    public Mutable<ILogicalExpression> getExpressionRef() {
+        return expression;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return visitor.transform(expression);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
new file mode 100644
index 0000000..a5b7074
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
@@ -0,0 +1,284 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class GroupByOperator extends AbstractOperatorWithNestedPlans {
+    // If the LogicalVariable in a pair is null, it means that the GroupBy is
+    // only grouping by the expression, without producing a new variable.
+    private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList;
+    private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList;
+
+    // In decorList, if the variable (first member of the pair) is null, the
+    // second member of the pair is variable reference which is propagated.
+
+    public GroupByOperator() {
+        super();
+        gByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+        decorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+    }
+
+    public GroupByOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList, List<ILogicalPlan> nestedPlans) {
+        super(nestedPlans);
+        this.decorList = decorList;
+        this.gByList = groupByList;
+    }
+
+    public void addGbyExpression(LogicalVariable variable, ILogicalExpression expression) {
+        this.gByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
+                new MutableObject<ILogicalExpression>(expression)));
+    }
+
+    public void addDecorExpression(LogicalVariable variable, ILogicalExpression expression) {
+        this.decorList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
+                new MutableObject<ILogicalExpression>(expression)));
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.GROUP;
+    }
+
+    public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getGroupByList() {
+        return gByList;
+    }
+
+    public String gByListToString() {
+        return veListToString(gByList);
+    }
+
+    public String decorListToString() {
+        return veListToString(decorList);
+    }
+
+    public List<LogicalVariable> getGbyVarList() {
+        List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size());
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) {
+            ILogicalExpression expr = ve.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) expr;
+                varList.add(v.getVariableReference());
+            }
+        }
+        return varList;
+    }
+
+    public List<LogicalVariable> getProducedGbyVarList() {
+        List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size());
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) {
+            varList.add(ve.first);
+        }
+        return varList;
+    }
+
+    public static String veListToString(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("[");
+        boolean fst = true;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
+            if (fst) {
+                fst = false;
+            } else {
+                sb.append("; ");
+            }
+            if (ve.first != null) {
+                sb.append(ve.first + " := " + ve.second);
+            } else {
+                sb.append(ve.second.getValue());
+            }
+        }
+        sb.append("]");
+        return sb.toString();
+    }
+
+    @Override
+    public void recomputeSchema() {
+        super.recomputeSchema();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            schema.add(p.first);
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+            schema.add(getDecorVariable(p));
+        }
+    }
+
+    @Override
+    public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+        // super.getProducedVariables(vars);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            if (p.first != null) {
+                vars.add(p.first);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+            if (p.first != null) {
+                vars.add(p.first);
+            }
+        }
+    }
+
+    @Override
+    public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : gByList) {
+            g.second.getValue().getUsedVariables(vars);
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : decorList) {
+            g.second.getValue().getUsedVariables(vars);
+        }
+        // super.getUsedVariables(vars);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+                    ILogicalExpression expr = p.second.getValue();
+                    if (p.first != null) {
+                        target.addVariable(p.first);
+                    } else {
+                        if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                            throw new AlgebricksException("hash group-by expects variable references.");
+                        }
+                        VariableReferenceExpression v = (VariableReferenceExpression) expr;
+                        target.addVariable(v.getVariableReference());
+                    }
+                }
+                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+                    ILogicalExpression expr = p.second.getValue();
+                    if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                        throw new AlgebricksException("pre-sorted group-by expects variable references.");
+                    }
+                    VariableReferenceExpression v = (VariableReferenceExpression) expr;
+                    LogicalVariable decor = v.getVariableReference();
+                    if (p.first != null) {
+                        target.addVariable(p.first);
+                    } else {
+                        target.addVariable(decor);
+                    }
+                }
+
+            }
+        };
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean b = false;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            if (visitor.transform(p.second)) {
+                b = true;
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+            if (visitor.transform(p.second)) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitGroupByOperator(this, arg);
+    }
+
+    public static LogicalVariable getDecorVariable(Pair<LogicalVariable, Mutable<ILogicalExpression>> p) {
+        if (p.first != null) {
+            return p.first;
+        } else {
+            VariableReferenceExpression e = (VariableReferenceExpression) p.second.getValue();
+            return e.getVariableReference();
+        }
+    }
+
+    public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getDecorList() {
+        return decorList;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        int n = 0;
+        for (ILogicalPlan p : nestedPlans) {
+            n += p.getRoots().size();
+        }
+        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
+        int i = 0;
+        for (ILogicalPlan p : nestedPlans) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
+                i++;
+            }
+        }
+        IVariableTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+        ILogicalOperator child = inputs.get(0).getValue();
+        IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(child);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getGroupByList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (p.first != null) {
+                env.setVarType(p.first, env2.getType(expr));
+                if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                    LogicalVariable v1 = ((VariableReferenceExpression) expr).getVariableReference();
+                    env.setVarType(v1, env2.getVarType(v1));
+                }
+            } else {
+                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
+                LogicalVariable v2 = vre.getVariableReference();
+                env.setVarType(v2, env2.getVarType(v2));
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (p.first != null) {
+                env.setVarType(p.first, env2.getType(expr));
+            } else {
+                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
+                LogicalVariable v2 = vre.getVariableReference();
+                env.setVarType(v2, env2.getVarType(v2));
+            }
+        }
+        return env;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
new file mode 100644
index 0000000..94b4252
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.Collection;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+
+/**
+ * @author rico
+ */
+public interface IOperatorExtension {
+
+    void setExecutionMode(ExecutionMode mode);
+
+    boolean isMap();
+
+    public IOperatorExtension newInstance();
+
+    boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException;
+
+    void setSchema(List<LogicalVariable> schema);
+
+    IPhysicalOperator getPhysicalOperator();
+
+    void setPhysicalOperator(IPhysicalOperator physicalOperator);
+
+    ExecutionMode getExecutionMode();
+    
+    public void getUsedVariables(Collection<LogicalVariable> usedVars);
+    
+    public void getProducedVariables(Collection<LogicalVariable> producedVars);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
new file mode 100644
index 0000000..4e50f4f
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IOperatorSchema extends Iterable<LogicalVariable> {
+    public void addAllVariables(IOperatorSchema source);
+
+    public void addAllNewVariables(IOperatorSchema source);
+
+    public int addVariable(LogicalVariable var);
+
+    public int findVariable(LogicalVariable var);
+
+    public LogicalVariable getVariable(int index);
+
+    public int getSize();
+
+    public void clear();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
new file mode 100644
index 0000000..c58215e
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class IndexInsertDeleteOperator extends AbstractLogicalOperator {
+
+    private final IDataSourceIndex<?, ?> dataSourceIndex;
+    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
+    // In the bulk-load case on ngram or keyword index,
+    // it contains [token, number of token] or [token].
+    // Otherwise, it contains secondary key information.
+    private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
+    private final Mutable<ILogicalExpression> filterExpr;
+    private final Kind operation;
+    private final boolean bulkload;
+    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+
+    public IndexInsertDeleteOperator(IDataSourceIndex<?, ?> dataSourceIndex,
+            List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
+            Mutable<ILogicalExpression> filterExpr, Kind operation, boolean bulkload) {
+        this.dataSourceIndex = dataSourceIndex;
+        this.primaryKeyExprs = primaryKeyExprs;
+        this.secondaryKeyExprs = secondaryKeyExprs;
+        this.filterExpr = filterExpr;
+        this.operation = operation;
+        this.bulkload = bulkload;
+    }
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean b = false;
+        for (int i = 0; i < primaryKeyExprs.size(); i++) {
+            if (visitor.transform(primaryKeyExprs.get(i))) {
+                b = true;
+            }
+        }
+        for (int i = 0; i < secondaryKeyExprs.size(); i++) {
+            if (visitor.transform(secondaryKeyExprs.get(i))) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitIndexInsertDeleteOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.INDEX_INSERT_DELETE;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
+        return primaryKeyExprs;
+    }
+
+    public IDataSourceIndex<?, ?> getDataSourceIndex() {
+        return dataSourceIndex;
+    }
+
+    public String getIndexName() {
+        return dataSourceIndex.getId().toString();
+    }
+
+    public List<Mutable<ILogicalExpression>> getSecondaryKeyExpressions() {
+        return secondaryKeyExprs;
+    }
+
+    public Mutable<ILogicalExpression> getFilterExpression() {
+        return filterExpr;
+    }
+
+    public Kind getOperation() {
+        return operation;
+    }
+
+    public boolean isBulkload() {
+        return bulkload;
+    }
+
+    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+        this.additionalFilteringExpressions = additionalFilteringExpressions;
+    }
+
+    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+        return additionalFilteringExpressions;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
new file mode 100644
index 0000000..e0a2cf1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class InnerJoinOperator extends AbstractBinaryJoinOperator {
+
+    public InnerJoinOperator(Mutable<ILogicalExpression> condition) {
+        super(JoinKind.INNER, condition);
+    }
+
+    public InnerJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
+            Mutable<ILogicalOperator> input2) {
+        super(JoinKind.INNER, condition, input1, input2);
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.INNERJOIN;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitInnerJoinOperator(this, arg);
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
new file mode 100644
index 0000000..134506c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class InsertDeleteOperator extends AbstractLogicalOperator {
+
+    public enum Kind {
+        INSERT,
+        DELETE
+    }
+
+    private final IDataSource<?> dataSource;
+
+    private final Mutable<ILogicalExpression> payloadExpr;
+
+    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
+
+    private final Kind operation;
+    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+
+    private final boolean bulkload;
+
+    public InsertDeleteOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payloadExpr,
+            List<Mutable<ILogicalExpression>> primaryKeyExprs, Kind operation, boolean bulkload) {
+        this.dataSource = dataSource;
+        this.payloadExpr = payloadExpr;
+        this.primaryKeyExprs = primaryKeyExprs;
+        this.operation = operation;
+        this.bulkload = bulkload;
+    }
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+        boolean changed = false;
+        changed = transform.transform(payloadExpr);
+        for (Mutable<ILogicalExpression> e : primaryKeyExprs) {
+            changed |= transform.transform(e);
+        }
+        return changed;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitInsertDeleteOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.INSERT_DELETE;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
+        return primaryKeyExprs;
+    }
+
+    public IDataSource<?> getDataSource() {
+        return dataSource;
+    }
+
+    public Mutable<ILogicalExpression> getPayloadExpression() {
+        return payloadExpr;
+    }
+
+    public Kind getOperation() {
+        return operation;
+    }
+
+    public boolean isBulkload() {
+        return bulkload;
+	}
+
+    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+        this.additionalFilteringExpressions = additionalFilteringExpressions;
+    }
+
+    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+        return additionalFilteringExpressions;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
new file mode 100644
index 0000000..1504039
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class LeftOuterJoinOperator extends AbstractBinaryJoinOperator {
+
+    public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition) {
+        super(JoinKind.LEFT_OUTER, condition);
+    }
+
+    public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
+            Mutable<ILogicalOperator> input2) {
+        super(JoinKind.LEFT_OUTER, condition, input1, input2);
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.LEFTOUTERJOIN;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitLeftOuterJoinOperator(this, arg);
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        int n = inputs.size();
+        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
+        for (int i = 0; i < n; i++) {
+            envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
+        }
+        PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
+        List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(inputs.get(1).getValue(), liveVars); // live variables from outer branch can be null together
+        env.getCorrelatedNullableVariableLists().add(liveVars);
+        return env;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
new file mode 100644
index 0000000..79145f4
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class LimitOperator extends AbstractLogicalOperator {
+
+    private final Mutable<ILogicalExpression> maxObjects; // mandatory
+    private final Mutable<ILogicalExpression> offset; // optional
+    private boolean topmost;
+
+    public LimitOperator(ILogicalExpression maxObjectsExpr, ILogicalExpression offsetExpr, boolean topmost) {
+        this.maxObjects = new MutableObject<ILogicalExpression>(maxObjectsExpr);
+        this.offset = new MutableObject<ILogicalExpression>(offsetExpr);
+        this.topmost = topmost;
+    }
+
+    public LimitOperator(ILogicalExpression maxObjectsExpr, boolean topmost) {
+        this(maxObjectsExpr, null, topmost);
+    }
+
+    public LimitOperator(ILogicalExpression maxObjects, ILogicalExpression offset) {
+        this(maxObjects, offset, true);
+    }
+
+    public LimitOperator(ILogicalExpression maxObjects) {
+        this(maxObjects, null, true);
+    }
+
+    public Mutable<ILogicalExpression> getMaxObjects() {
+        return maxObjects;
+    }
+
+    public Mutable<ILogicalExpression> getOffset() {
+        return offset;
+    }
+
+    public boolean isTopmostLimitOp() {
+        return topmost;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitLimitOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean b = false;
+        if (visitor.transform(maxObjects)) {
+            b = true;
+        }
+        if (offset.getValue() != null) {
+            if (visitor.transform(offset)) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.LIMIT;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return true;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java
new file mode 100644
index 0000000..5cecf07
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class MaterializeOperator extends AbstractLogicalOperator {
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.MATERIALIZE;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitMaterializeOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
new file mode 100644
index 0000000..f6f62b6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class NestedTupleSourceOperator extends AbstractLogicalOperator {
+    private Mutable<ILogicalOperator> dataSourceReference;
+
+    public NestedTupleSourceOperator(Mutable<ILogicalOperator> dataSourceReference) {
+        this.dataSourceReference = dataSourceReference;
+    }
+
+    public ILogicalOperator getSourceOperator() {
+        return dataSourceReference.getValue().getInputs().get(0).getValue();
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.NESTEDTUPLESOURCE;
+    }
+
+    public Mutable<ILogicalOperator> getDataSourceReference() {
+        return dataSourceReference;
+    }
+
+    public void setDataSourceReference(Mutable<ILogicalOperator> dataSourceReference) {
+        this.dataSourceReference = dataSourceReference;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        ILogicalOperator topOp = dataSourceReference.getValue();
+        for (Mutable<ILogicalOperator> i : topOp.getInputs()) {
+            schema.addAll(i.getValue().getSchema());
+        }
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+        // do nothing
+        return false;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitNestedTupleSourceOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException {
+        ITypeEnvPointer[] p = new ITypeEnvPointer[1];
+        p[0] = new ITypeEnvPointer() {
+
+            @Override
+            public IVariableTypeEnvironment getTypeEnv() {
+                ILogicalOperator op = dataSourceReference.getValue().getInputs().get(0).getValue();
+                return ctx.getOutputTypeEnvironment(op);
+            }
+        };
+        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+                ctx.getMetadataProvider(), TypePropagationPolicy.ALL, p);
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return computeOutputTypeEnvironment(ctx);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
new file mode 100644
index 0000000..335fb36
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class OrderOperator extends AbstractLogicalOperator {
+
+    public interface IOrder {
+        public enum OrderKind {
+            FUNCTIONCALL,
+            ASC,
+            DESC
+        };
+
+        public Mutable<ILogicalExpression> getExpressionRef();
+
+        public OrderKind getKind();
+    }
+
+    public static IOrder ASC_ORDER = new IOrder() {
+
+        @Override
+        public Mutable<ILogicalExpression> getExpressionRef() {
+            return null;
+        }
+
+        @Override
+        public OrderKind getKind() {
+            return OrderKind.ASC;
+        }
+
+    };
+
+    public static IOrder DESC_ORDER = new IOrder() {
+
+        @Override
+        public Mutable<ILogicalExpression> getExpressionRef() {
+            return null;
+        }
+
+        @Override
+        public OrderKind getKind() {
+            return OrderKind.DESC;
+        }
+    };
+
+    public class FunOrder implements IOrder {
+        private final Mutable<ILogicalExpression> f;
+
+        public FunOrder(Mutable<ILogicalExpression> f) {
+            this.f = f;
+        }
+
+        @Override
+        public Mutable<ILogicalExpression> getExpressionRef() {
+            return f;
+        }
+
+        @Override
+        public OrderKind getKind() {
+            return OrderKind.FUNCTIONCALL;
+        }
+
+    };
+
+    private final List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions;
+
+    // These are pairs of type (comparison, expr) where comparison is
+    // ASC or DESC or a boolean function of arity 2 that can take as
+    // arguments results of expr.
+
+    public OrderOperator() {
+        orderExpressions = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+    }
+
+    public OrderOperator(List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions) {
+        this.orderExpressions = orderExpressions;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.ORDER;
+    }
+
+    public List<Pair<IOrder, Mutable<ILogicalExpression>>> getOrderExpressions() {
+        return orderExpressions;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean b = false;
+        for (Pair<IOrder, Mutable<ILogicalExpression>> p : orderExpressions) {
+            if (p.first.getKind() == OrderKind.FUNCTIONCALL) {
+                FunOrder fo = (FunOrder) p.first;
+                Mutable<ILogicalExpression> r1 = fo.getExpressionRef();
+                if (visitor.transform(r1)) {
+                    b = true;
+                }
+            }
+            if (visitor.transform(p.second)) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitOrderOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
new file mode 100644
index 0000000..a6c8f6f
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.PartitioningSplitOperatorDescriptor;
+
+/**
+ * Partitions it's input based on a given list of expressions.
+ * Each expression is assumed to return true/false,
+ * and there is exactly one output branch per expression (optionally, plus one default branch).
+ * For each input tuple, the expressions are evaluated one-by-one,
+ * and the tuple is written to first output branch whose corresponding
+ * expression evaluates to true.
+ * If all expressions evaluate to false, then
+ * the tuple is written to the default output branch, if any was given.
+ * If no output branch was given, then such tuples are simply dropped.
+ * Given N expressions there may be N or N+1 output branches because the default output branch may be separate from the regular output branches.
+ */
+public class PartitioningSplitOperator extends AbstractLogicalOperator {
+
+    private final List<Mutable<ILogicalExpression>> expressions;
+    private final int defaultBranchIndex;
+
+    public PartitioningSplitOperator(List<Mutable<ILogicalExpression>> expressions, int defaultBranchIndex) throws AlgebricksException {
+        this.expressions = expressions;
+        this.defaultBranchIndex = defaultBranchIndex;
+        // Check that the default output branch index is in [0, N], where N is the number of expressions.
+        if (defaultBranchIndex != PartitioningSplitOperatorDescriptor.NO_DEFAULT_BRANCH
+                && defaultBranchIndex > expressions.size()) {
+            throw new AlgebricksException("Default branch index out of bounds. Number of exprs given: "
+                    + expressions.size() + ". The maximum default branch index may therefore be: " + expressions.size());
+        }
+    }
+
+    public List<Mutable<ILogicalExpression>> getExpressions() {
+        return expressions;
+    }
+
+    public int getDefaultBranchIndex() {
+        return defaultBranchIndex;
+    }
+    
+    public int getNumOutputBranches() {
+        return (defaultBranchIndex == expressions.size()) ? expressions.size() + 1 : expressions.size();
+    }
+    
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.PARTITIONINGSPLIT;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean b = false;
+        for (int i = 0; i < expressions.size(); i++) {
+            if (visitor.transform(expressions.get(i))) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitPartitioningSplitOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
new file mode 100644
index 0000000..bcba280
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FilteredVariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ProjectOperator extends AbstractLogicalOperator {
+
+    private final List<LogicalVariable> variables;
+
+    public ProjectOperator(List<LogicalVariable> variables) {
+        this.variables = variables;
+    }
+
+    public ProjectOperator(LogicalVariable v) {
+        this.variables = new ArrayList<LogicalVariable>(1);
+        this.getVariables().add(v);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+        // do nothing
+        return false;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.PROJECT;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>(getVariables());
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new FilteredVariablePropagationPolicy(getVariables());
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitProjectOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return true;
+    }
+
+    public List<LogicalVariable> getVariables() {
+        return variables;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
new file mode 100644
index 0000000..2c5bd54
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ReplicateOperator extends AbstractLogicalOperator {
+
+    private int outputArity = 2;
+    private boolean[] outputMaterializationFlags = new boolean[outputArity];
+    private List<Mutable<ILogicalOperator>> outputs;
+
+    public ReplicateOperator(int outputArity) {
+        this.outputArity = outputArity;
+        this.outputMaterializationFlags = new boolean[outputArity];
+        this.outputs = new ArrayList<Mutable<ILogicalOperator>>();
+    }
+
+    public ReplicateOperator(int outputArity, boolean[] outputMaterializationFlags) {
+        this.outputArity = outputArity;
+        this.outputMaterializationFlags = outputMaterializationFlags;
+        this.outputs = new ArrayList<Mutable<ILogicalOperator>>();
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.REPLICATE;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitReplicateOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+    }
+
+    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+        // do nothing
+    }
+
+    public int getOutputArity() {
+        return outputArity;
+    }
+
+    public int setOutputArity(int outputArity) {
+        return this.outputArity = outputArity;
+    }
+
+    public void setOutputMaterializationFlags(boolean[] outputMaterializationFlags) {
+        this.outputMaterializationFlags = outputMaterializationFlags;
+    }
+
+    public boolean[] getOutputMaterializationFlags() {
+        return outputMaterializationFlags;
+    }
+
+    public List<Mutable<ILogicalOperator>> getOutputs() {
+        return outputs;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    public boolean isBlocker() {
+        for (boolean requiresMaterialization : outputMaterializationFlags) {
+            if (requiresMaterialization) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
new file mode 100644
index 0000000..4f608bb
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class RunningAggregateOperator extends AbstractAssignOperator {
+
+    public RunningAggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+        super(variables, expressions);
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.RUNNINGAGGREGATE;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitRunningAggregateOperator(this, arg);
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                target.addAllVariables(sources[0]);
+                for (LogicalVariable v : variables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            env.setVarType(
+                    variables.get(i),
+                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
+                            env));
+        }
+        return env;
+    }
+
+}

[50/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
deleted file mode 100644
index 39f3550..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.constraints;
-
-public class AlgebricksAbsolutePartitionConstraint extends AlgebricksPartitionConstraint {
-    private final String[] locations;
-
-    public AlgebricksAbsolutePartitionConstraint(String[] locations) {
-        this.locations = locations;
-    }
-
-    @Override
-    public PartitionConstraintType getPartitionConstraintType() {
-        return PartitionConstraintType.ABSOLUTE;
-    }
-
-    public String[] getLocations() {
-        return locations;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
deleted file mode 100644
index dd62f47..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.constraints;
-
-public class AlgebricksCountPartitionConstraint extends AlgebricksPartitionConstraint {
-
-    private final int count;
-
-    public AlgebricksCountPartitionConstraint(int count) {
-        this.count = count;
-    }
-
-    @Override
-    public PartitionConstraintType getPartitionConstraintType() {
-        return PartitionConstraintType.COUNT;
-    }
-
-    public int getCount() {
-        return count;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
deleted file mode 100644
index a458451..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.constraints;
-
-public abstract class AlgebricksPartitionConstraint {
-    public enum PartitionConstraintType {
-        ABSOLUTE,
-        COUNT
-    }
-
-    public abstract PartitionConstraintType getPartitionConstraintType();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
deleted file mode 100644
index 8ad27b1..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.constraints;
-
-import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class AlgebricksPartitionConstraintHelper {
-
-    public static void setPartitionConstraintInJobSpec(JobSpecification jobSpec, IOperatorDescriptor opDesc,
-            AlgebricksPartitionConstraint apc) {
-        switch (apc.getPartitionConstraintType()) {
-            case ABSOLUTE: {
-                AlgebricksAbsolutePartitionConstraint absPc = (AlgebricksAbsolutePartitionConstraint) apc;
-                PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, opDesc, absPc.getLocations());
-                break;
-            }
-            case COUNT: {
-                AlgebricksCountPartitionConstraint cntPc = (AlgebricksCountPartitionConstraint) apc;
-                PartitionConstraintHelper.addPartitionCountConstraint(jobSpec, opDesc, cntPc.getCount());
-                break;
-            }
-            default: {
-                throw new IllegalStateException();
-            }
-        }
-    }
-
-    public static int getPartitionCount(AlgebricksPartitionConstraint partitionConstraint) {
-        switch (partitionConstraint.getPartitionConstraintType()) {
-            case COUNT: {
-                AlgebricksCountPartitionConstraint pcc = (AlgebricksCountPartitionConstraint) partitionConstraint;
-                return pcc.getCount();
-            }
-            case ABSOLUTE: {
-                AlgebricksAbsolutePartitionConstraint epc = (AlgebricksAbsolutePartitionConstraint) partitionConstraint;
-                return epc.getLocations().length;
-            }
-            default: {
-                throw new IllegalStateException();
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java
deleted file mode 100644
index 6f39167..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/AlgebricksException.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.exceptions;
-
-public class AlgebricksException extends Exception {
-    private static final long serialVersionUID = 1L;
-
-    public AlgebricksException() {
-    }
-
-    public AlgebricksException(String message) {
-        super(message);
-    }
-
-    public AlgebricksException(Throwable cause) {
-        super(cause);
-    }
-
-    public AlgebricksException(String message, Throwable cause) {
-        super(message, cause);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java
deleted file mode 100644
index cc0c975..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/exceptions/NotImplementedException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.exceptions;
-
-public class NotImplementedException extends RuntimeException {
-    private static final long serialVersionUID = 2L;
-
-    public NotImplementedException() {
-        System.err.println("Not implemented.");
-    }
-
-    public NotImplementedException(String message) {
-        super(message);
-    }
-
-    public NotImplementedException(Throwable cause) {
-        super(cause);
-    }
-
-    public NotImplementedException(String message, Throwable cause) {
-        super(message, cause);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java
deleted file mode 100644
index f4ed6c1..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/ListSet.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.utils;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-public class ListSet<E> implements Set<E> {
-    private List<E> list = new ArrayList<E>();
-
-    public ListSet() {
-    }
-
-    public ListSet(Collection<? extends E> arg) {
-        this.addAll(arg);
-    }
-
-    @Override
-    public boolean add(E arg0) {
-        if (list.contains(arg0))
-            return false;
-        return list.add(arg0);
-    }
-
-    @Override
-    public boolean addAll(Collection<? extends E> arg0) {
-        for (E item : arg0)
-            if (list.contains(item))
-                return false;
-        return list.addAll(arg0);
-    }
-
-    @Override
-    public void clear() {
-        list.clear();
-    }
-
-    @Override
-    public boolean contains(Object arg0) {
-        return list.contains(arg0);
-    }
-
-    @Override
-    public boolean containsAll(Collection<?> arg0) {
-        return list.containsAll(arg0);
-    }
-
-    @Override
-    public boolean isEmpty() {
-        return list.isEmpty();
-    }
-
-    @Override
-    public Iterator<E> iterator() {
-        return list.iterator();
-    }
-
-    @Override
-    public boolean remove(Object arg0) {
-        return list.remove(arg0);
-    }
-
-    @Override
-    public boolean removeAll(Collection<?> arg0) {
-        return list.removeAll(arg0);
-    }
-
-    @Override
-    public boolean retainAll(Collection<?> arg0) {
-        return list.retainAll(arg0);
-    }
-
-    @Override
-    public int size() {
-        return list.size();
-    }
-
-    @Override
-    public Object[] toArray() {
-        return list.toArray();
-    }
-
-    @Override
-    public <T> T[] toArray(T[] arg0) {
-        return list.toArray(arg0);
-    }
-
-    @SuppressWarnings("unchecked")
-    @Override
-    public boolean equals(Object arg) {
-        if (!(arg instanceof Set))
-            return false;
-        Set<E> set = (Set<E>) arg;
-        for (E item : set) {
-            if (!this.contains(item))
-                return false;
-        }
-        for (E item : this) {
-            if (!set.contains(item))
-                return false;
-        }
-        return true;
-    }
-
-    @Override
-    public String toString() {
-        return list.toString();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java
deleted file mode 100644
index e8ba583..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Pair.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.utils;
-
-import java.io.Serializable;
-
-public class Pair<T1, T2> implements Serializable {
-
-    private static final long serialVersionUID = 1L;
-    public T1 first;
-    public T2 second;
-
-    public Pair(T1 first, T2 second) {
-        this.first = first;
-        this.second = second;
-    }
-
-    @Override
-    public String toString() {
-        return first + "," + second;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof Pair<?, ?>)) {
-            return false;
-        } else {
-            Pair<?, ?> p = (Pair<?, ?>) obj;
-            return this.first.equals(p.first) && this.second.equals(p.second);
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        return first.hashCode() * 31 + second.hashCode();
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java b/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java
deleted file mode 100644
index fe76c4a..0000000
--- a/algebricks/algebricks-common/src/main/java/edu/uci/ics/hyracks/algebricks/common/utils/Triple.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.common.utils;
-
-public class Triple<T1, T2, T3> {
-    public T1 first;
-    public T2 second;
-    public T3 third;
-
-    public Triple(T1 first, T2 second, T3 third) {
-        this.first = first;
-        this.second = second;
-        this.third = third;
-    }
-
-    @Override
-    public String toString() {
-        return first + "," + second + ", " + third;
-    }
-
-    @Override
-    public int hashCode() {
-        return first.hashCode() * 31 + second.hashCode() * 15 + third.hashCode();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (!(o instanceof Triple<?, ?, ?>))
-            return false;
-        Triple<?, ?, ?> triple = (Triple<?, ?, ?>) o;
-        return first.equals(triple.first) && second.equals(triple.second) && third.equals(triple.third);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
new file mode 100644
index 0000000..39f3550
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksAbsolutePartitionConstraint.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.constraints;
+
+public class AlgebricksAbsolutePartitionConstraint extends AlgebricksPartitionConstraint {
+    private final String[] locations;
+
+    public AlgebricksAbsolutePartitionConstraint(String[] locations) {
+        this.locations = locations;
+    }
+
+    @Override
+    public PartitionConstraintType getPartitionConstraintType() {
+        return PartitionConstraintType.ABSOLUTE;
+    }
+
+    public String[] getLocations() {
+        return locations;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
new file mode 100644
index 0000000..dd62f47
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksCountPartitionConstraint.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.constraints;
+
+public class AlgebricksCountPartitionConstraint extends AlgebricksPartitionConstraint {
+
+    private final int count;
+
+    public AlgebricksCountPartitionConstraint(int count) {
+        this.count = count;
+    }
+
+    @Override
+    public PartitionConstraintType getPartitionConstraintType() {
+        return PartitionConstraintType.COUNT;
+    }
+
+    public int getCount() {
+        return count;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
new file mode 100644
index 0000000..a458451
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraint.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.constraints;
+
+public abstract class AlgebricksPartitionConstraint {
+    public enum PartitionConstraintType {
+        ABSOLUTE,
+        COUNT
+    }
+
+    public abstract PartitionConstraintType getPartitionConstraintType();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
new file mode 100644
index 0000000..8ad27b1
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/constraints/AlgebricksPartitionConstraintHelper.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.constraints;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class AlgebricksPartitionConstraintHelper {
+
+    public static void setPartitionConstraintInJobSpec(JobSpecification jobSpec, IOperatorDescriptor opDesc,
+            AlgebricksPartitionConstraint apc) {
+        switch (apc.getPartitionConstraintType()) {
+            case ABSOLUTE: {
+                AlgebricksAbsolutePartitionConstraint absPc = (AlgebricksAbsolutePartitionConstraint) apc;
+                PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, opDesc, absPc.getLocations());
+                break;
+            }
+            case COUNT: {
+                AlgebricksCountPartitionConstraint cntPc = (AlgebricksCountPartitionConstraint) apc;
+                PartitionConstraintHelper.addPartitionCountConstraint(jobSpec, opDesc, cntPc.getCount());
+                break;
+            }
+            default: {
+                throw new IllegalStateException();
+            }
+        }
+    }
+
+    public static int getPartitionCount(AlgebricksPartitionConstraint partitionConstraint) {
+        switch (partitionConstraint.getPartitionConstraintType()) {
+            case COUNT: {
+                AlgebricksCountPartitionConstraint pcc = (AlgebricksCountPartitionConstraint) partitionConstraint;
+                return pcc.getCount();
+            }
+            case ABSOLUTE: {
+                AlgebricksAbsolutePartitionConstraint epc = (AlgebricksAbsolutePartitionConstraint) partitionConstraint;
+                return epc.getLocations().length;
+            }
+            default: {
+                throw new IllegalStateException();
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
new file mode 100644
index 0000000..6f39167
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/AlgebricksException.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.exceptions;
+
+public class AlgebricksException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public AlgebricksException() {
+    }
+
+    public AlgebricksException(String message) {
+        super(message);
+    }
+
+    public AlgebricksException(Throwable cause) {
+        super(cause);
+    }
+
+    public AlgebricksException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/NotImplementedException.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/NotImplementedException.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/NotImplementedException.java
new file mode 100644
index 0000000..cc0c975
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/exceptions/NotImplementedException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.exceptions;
+
+public class NotImplementedException extends RuntimeException {
+    private static final long serialVersionUID = 2L;
+
+    public NotImplementedException() {
+        System.err.println("Not implemented.");
+    }
+
+    public NotImplementedException(String message) {
+        super(message);
+    }
+
+    public NotImplementedException(Throwable cause) {
+        super(cause);
+    }
+
+    public NotImplementedException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/ListSet.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/ListSet.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/ListSet.java
new file mode 100644
index 0000000..f4ed6c1
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/ListSet.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.utils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+public class ListSet<E> implements Set<E> {
+    private List<E> list = new ArrayList<E>();
+
+    public ListSet() {
+    }
+
+    public ListSet(Collection<? extends E> arg) {
+        this.addAll(arg);
+    }
+
+    @Override
+    public boolean add(E arg0) {
+        if (list.contains(arg0))
+            return false;
+        return list.add(arg0);
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends E> arg0) {
+        for (E item : arg0)
+            if (list.contains(item))
+                return false;
+        return list.addAll(arg0);
+    }
+
+    @Override
+    public void clear() {
+        list.clear();
+    }
+
+    @Override
+    public boolean contains(Object arg0) {
+        return list.contains(arg0);
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> arg0) {
+        return list.containsAll(arg0);
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return list.isEmpty();
+    }
+
+    @Override
+    public Iterator<E> iterator() {
+        return list.iterator();
+    }
+
+    @Override
+    public boolean remove(Object arg0) {
+        return list.remove(arg0);
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> arg0) {
+        return list.removeAll(arg0);
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> arg0) {
+        return list.retainAll(arg0);
+    }
+
+    @Override
+    public int size() {
+        return list.size();
+    }
+
+    @Override
+    public Object[] toArray() {
+        return list.toArray();
+    }
+
+    @Override
+    public <T> T[] toArray(T[] arg0) {
+        return list.toArray(arg0);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public boolean equals(Object arg) {
+        if (!(arg instanceof Set))
+            return false;
+        Set<E> set = (Set<E>) arg;
+        for (E item : set) {
+            if (!this.contains(item))
+                return false;
+        }
+        for (E item : this) {
+            if (!set.contains(item))
+                return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return list.toString();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java
new file mode 100644
index 0000000..e8ba583
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Pair.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.utils;
+
+import java.io.Serializable;
+
+public class Pair<T1, T2> implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+    public T1 first;
+    public T2 second;
+
+    public Pair(T1 first, T2 second) {
+        this.first = first;
+        this.second = second;
+    }
+
+    @Override
+    public String toString() {
+        return first + "," + second;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof Pair<?, ?>)) {
+            return false;
+        } else {
+            Pair<?, ?> p = (Pair<?, ?>) obj;
+            return this.first.equals(p.first) && this.second.equals(p.second);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return first.hashCode() * 31 + second.hashCode();
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java
new file mode 100644
index 0000000..fe76c4a
--- /dev/null
+++ b/algebricks/algebricks-common/src/main/java/org/apache/hyracks/algebricks/common/utils/Triple.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.common.utils;
+
+public class Triple<T1, T2, T3> {
+    public T1 first;
+    public T2 second;
+    public T3 third;
+
+    public Triple(T1 first, T2 second, T3 third) {
+        this.first = first;
+        this.second = second;
+        this.third = third;
+    }
+
+    @Override
+    public String toString() {
+        return first + "," + second + ", " + third;
+    }
+
+    @Override
+    public int hashCode() {
+        return first.hashCode() * 31 + second.hashCode() * 15 + third.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof Triple<?, ?, ?>))
+            return false;
+        Triple<?, ?, ?> triple = (Triple<?, ?, ?>) o;
+        return first.equals(triple.first) && second.equals(triple.second) && third.equals(triple.third);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
deleted file mode 100644
index 3d92d68..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.api;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFamilyProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
-
-public abstract class AbstractCompilerFactoryBuilder {
-
-    protected List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
-    protected List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
-    protected ITypeTraitProvider typeTraitProvider;
-    protected ISerializerDeserializerProvider serializerDeserializerProvider;
-    protected IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
-    protected IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider;
-    protected IBinaryComparatorFactoryProvider comparatorFactoryProvider;
-    protected IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
-    protected IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory;
-    protected IPrinterFactoryProvider printerProvider;
-    protected IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider;
-    protected IExpressionRuntimeProvider expressionRuntimeProvider;
-    protected IExpressionTypeComputer expressionTypeComputer;
-    protected INullableTypeComputer nullableTypeComputer;
-    protected IExpressionEvalSizeComputer expressionEvalSizeComputer;
-    protected INullWriterFactory nullWriterFactory;
-    protected INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
-    protected IPartialAggregationTypeComputer partialAggregationTypeComputer;
-    protected IMergeAggregationExpressionFactory mergeAggregationExpressionFactory;
-    protected PhysicalOptimizationConfig physicalOptimizationConfig = new PhysicalOptimizationConfig();
-    protected AlgebricksPartitionConstraint clusterLocations;
-
-    public abstract ICompilerFactory create();
-
-    public void setLogicalRewrites(List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites) {
-        this.logicalRewrites = logicalRewrites;
-    }
-
-    public void setPhysicalRewrites(List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites) {
-        this.physicalRewrites = physicalRewrites;
-    }
-
-    public void setTypeTraitProvider(ITypeTraitProvider typeTraitProvider) {
-        this.typeTraitProvider = typeTraitProvider;
-    }
-
-    public ITypeTraitProvider getTypeTraitProvider() {
-        return typeTraitProvider;
-    }
-
-    public void setSerializerDeserializerProvider(ISerializerDeserializerProvider serializerDeserializerProvider) {
-        this.serializerDeserializerProvider = serializerDeserializerProvider;
-    }
-
-    public ISerializerDeserializerProvider getSerializerDeserializerProvider() {
-        return serializerDeserializerProvider;
-    }
-
-    public void setHashFunctionFactoryProvider(IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider) {
-        this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
-    }
-
-    public IBinaryHashFunctionFactoryProvider getHashFunctionFactoryProvider() {
-        return hashFunctionFactoryProvider;
-    }
-
-    public void setHashFunctionFamilyProvider(IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider) {
-        this.hashFunctionFamilyProvider = hashFunctionFamilyProvider;
-    }
-
-    public IBinaryHashFunctionFamilyProvider getHashFunctionFamilyProvider() {
-        return hashFunctionFamilyProvider;
-    }
-
-    public void setComparatorFactoryProvider(IBinaryComparatorFactoryProvider comparatorFactoryProvider) {
-        this.comparatorFactoryProvider = comparatorFactoryProvider;
-    }
-
-    public IBinaryComparatorFactoryProvider getComparatorFactoryProvider() {
-        return comparatorFactoryProvider;
-    }
-    
-    public void setPredicateEvaluatorFactoryProvider(IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider) {
-        this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
-    }
-
-    public IPredicateEvaluatorFactoryProvider getPredicateEvaluatorFactory() {
-        return predEvaluatorFactoryProvider;
-    }
-
-    public void setBinaryBooleanInspectorFactory(IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
-        this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
-    }
-
-    public IBinaryBooleanInspectorFactory getBinaryBooleanInspectorFactory() {
-        return binaryBooleanInspectorFactory;
-    }
-
-    public void setBinaryIntegerInspectorFactory(IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory) {
-        this.binaryIntegerInspectorFactory = binaryIntegerInspectorFactory;
-    }
-
-    public IBinaryIntegerInspectorFactory getBinaryIntegerInspectorFactory() {
-        return binaryIntegerInspectorFactory;
-    }
-
-    public void setPrinterProvider(IPrinterFactoryProvider printerProvider) {
-        this.printerProvider = printerProvider;
-    }
-
-    public IPrinterFactoryProvider getPrinterProvider() {
-        return printerProvider;
-    }
-
-    public void setExpressionRuntimeProvider(IExpressionRuntimeProvider expressionRuntimeProvider) {
-        this.expressionRuntimeProvider = expressionRuntimeProvider;
-    }
-
-    public IExpressionRuntimeProvider getExpressionRuntimeProvider() {
-        return expressionRuntimeProvider;
-    }
-
-    public void setExpressionTypeComputer(IExpressionTypeComputer expressionTypeComputer) {
-        this.expressionTypeComputer = expressionTypeComputer;
-    }
-
-    public IExpressionTypeComputer getExpressionTypeComputer() {
-        return expressionTypeComputer;
-    }
-
-    public void setClusterLocations(AlgebricksPartitionConstraint clusterLocations) {
-        this.clusterLocations = clusterLocations;
-    }
-
-    public AlgebricksPartitionConstraint getClusterLocations() {
-        return clusterLocations;
-    }
-
-    public void setNullWriterFactory(INullWriterFactory nullWriterFactory) {
-        this.nullWriterFactory = nullWriterFactory;
-    }
-
-    public INullWriterFactory getNullWriterFactory() {
-        return nullWriterFactory;
-    }
-
-    public void setExpressionEvalSizeComputer(IExpressionEvalSizeComputer expressionEvalSizeComputer) {
-        this.expressionEvalSizeComputer = expressionEvalSizeComputer;
-    }
-
-    public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
-        return expressionEvalSizeComputer;
-    }
-
-    public void setNormalizedKeyComputerFactoryProvider(
-            INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider) {
-        this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
-    }
-
-    public INormalizedKeyComputerFactoryProvider getNormalizedKeyComputerFactoryProvider() {
-        return normalizedKeyComputerFactoryProvider;
-    }
-
-    public IPartialAggregationTypeComputer getPartialAggregationTypeComputer() {
-        return partialAggregationTypeComputer;
-    }
-
-    public void setPartialAggregationTypeComputer(IPartialAggregationTypeComputer partialAggregationTypeComputer) {
-        this.partialAggregationTypeComputer = partialAggregationTypeComputer;
-    }
-
-    public IMergeAggregationExpressionFactory getIMergeAggregationExpressionFactory() {
-        return mergeAggregationExpressionFactory;
-    }
-
-    public void setIMergeAggregationExpressionFactory(
-            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory) {
-        this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
-    }
-
-    public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
-        return physicalOptimizationConfig;
-    }
-
-    public void setPhysicalOptimizationConfig(PhysicalOptimizationConfig physicalOptimizationConfig) {
-        this.physicalOptimizationConfig = physicalOptimizationConfig;
-    }
-
-    public void setNullableTypeComputer(INullableTypeComputer nullableTypeComputer) {
-        this.nullableTypeComputer = nullableTypeComputer;
-    }
-
-    public INullableTypeComputer getNullableTypeComputer() {
-        return nullableTypeComputer;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
deleted file mode 100644
index 26dc343..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.api;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuilder {
-
-    public static class DefaultOptimizationContextFactory implements IOptimizationContextFactory {
-
-        public static final DefaultOptimizationContextFactory INSTANCE = new DefaultOptimizationContextFactory();
-
-        private DefaultOptimizationContextFactory() {
-        }
-
-        @Override
-        public IOptimizationContext createOptimizationContext(int varCounter,
-                IExpressionEvalSizeComputer expressionEvalSizeComputer,
-                IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
-                IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
-                PhysicalOptimizationConfig physicalOptimizationConfig) {
-            LogicalOperatorPrettyPrintVisitor prettyPrintVisitor = new LogicalOperatorPrettyPrintVisitor();
-            return new AlgebricksOptimizationContext(varCounter, expressionEvalSizeComputer,
-                    mergeAggregationExpressionFactory, expressionTypeComputer, nullableTypeComputer,
-                    physicalOptimizationConfig, prettyPrintVisitor);
-        }
-    }
-
-    private IOptimizationContextFactory optCtxFactory;
-
-    public HeuristicCompilerFactoryBuilder() {
-        this.optCtxFactory = DefaultOptimizationContextFactory.INSTANCE;
-    }
-
-    public HeuristicCompilerFactoryBuilder(IOptimizationContextFactory optCtxFactory) {
-        this.optCtxFactory = optCtxFactory;
-    }
-
-    @Override
-    public ICompilerFactory create() {
-        return new ICompilerFactory() {
-            @Override
-            public ICompiler createCompiler(final ILogicalPlan plan, final IMetadataProvider<?, ?> metadata,
-                    int varCounter) {
-                final IOptimizationContext oc = optCtxFactory.createOptimizationContext(varCounter,
-                        expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
-                        nullableTypeComputer, physicalOptimizationConfig);
-                oc.setMetadataDeclarations(metadata);
-                final HeuristicOptimizer opt = new HeuristicOptimizer(plan, logicalRewrites, physicalRewrites, oc);
-                return new ICompiler() {
-
-                    @Override
-                    public void optimize() throws AlgebricksException {
-                        opt.optimize();
-                    }
-
-                    @Override
-                    public JobSpecification createJob(Object appContext,
-                            IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException {
-                        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting Job Generation.\n");
-                        JobGenContext context = new JobGenContext(null, metadata, appContext,
-                                serializerDeserializerProvider, hashFunctionFactoryProvider,
-                                hashFunctionFamilyProvider, comparatorFactoryProvider, typeTraitProvider,
-                                binaryBooleanInspectorFactory, binaryIntegerInspectorFactory, printerProvider,
-                                nullWriterFactory, normalizedKeyComputerFactoryProvider, expressionRuntimeProvider,
-                                expressionTypeComputer, nullableTypeComputer, oc, expressionEvalSizeComputer,
-                                partialAggregationTypeComputer, predEvaluatorFactoryProvider,
-                                physicalOptimizationConfig.getFrameSize(), clusterLocations);
-
-                        PlanCompiler pc = new PlanCompiler(context);
-                        return pc.compilePlan(plan, null, jobEventListenerFactory);
-                    }
-                };
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
deleted file mode 100644
index 436be40..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.api;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public interface ICompiler {
-    public void optimize() throws AlgebricksException;
-
-    public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java
deleted file mode 100644
index 66f3b81..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.api;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public interface ICompilerFactory {
-    ICompiler createCompiler(ILogicalPlan plan, IMetadataProvider<?, ?> metadata, int varCounter);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
deleted file mode 100644
index 2062c5f..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
-
-import java.util.Collection;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * 
- * Runs each rule until it produces no changes. Then the whole collection of
- * rules is run again until no change is made.
- * 
- * 
- * @author Nicola
- * 
- */
-
-public class PrioritizedRuleController extends AbstractRuleController {
-
-    public PrioritizedRuleController() {
-        super();
-    }
-
-    @Override
-    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
-            throws AlgebricksException {
-        boolean anyRuleFired = false;
-        boolean anyChange = false;
-        do {
-            anyChange = false;
-            for (IAlgebraicRewriteRule r : rules) {
-                while (true) {
-                    boolean ruleFired = rewriteOperatorRef(root, r);
-                    if (ruleFired) {
-                        anyChange = true;
-                        anyRuleFired = true;
-                    } else {
-                        break; // go to next rule
-                    }
-                }
-            }
-        } while (anyChange);
-        return anyRuleFired;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
deleted file mode 100644
index 2f65c8a..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
-
-import java.util.Collection;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Runs rules sequentially (round-robin), until one iteration over all rules
- * produces no change.
- * 
- * @author Nicola
- */
-public class SequentialFixpointRuleController extends AbstractRuleController {
-
-    private boolean fullDfs;
-
-    public SequentialFixpointRuleController(boolean fullDfs) {
-        super();
-        this.fullDfs = fullDfs;
-    }
-
-    @Override
-    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
-            Collection<IAlgebraicRewriteRule> ruleCollection) throws AlgebricksException {
-        boolean anyRuleFired = false;
-        boolean anyChange = false;
-        do {
-            anyChange = false;
-            for (IAlgebraicRewriteRule rule : ruleCollection) {
-                boolean ruleFired = rewriteOperatorRef(root, rule, true, fullDfs);
-                if (ruleFired) {
-                    anyChange = true;
-                    anyRuleFired = true;
-                }
-            }
-        } while (anyChange);
-        return anyRuleFired;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java b/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
deleted file mode 100644
index 0f8ec80..0000000
--- a/algebricks/algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
-
-import java.util.Collection;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class SequentialOnceRuleController extends AbstractRuleController {
-
-    private final boolean enterNestedPlans;
-
-    public SequentialOnceRuleController(boolean enterNestedPlans) {
-        super();
-        this.enterNestedPlans = enterNestedPlans;
-    }
-
-    @Override
-    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
-            throws AlgebricksException {
-        boolean fired = false;
-        for (IAlgebraicRewriteRule rule : rules) {
-            if (rewriteOperatorRef(root, rule, enterNestedPlans, true)) {
-                fired = true;
-            }
-        }
-        return fired;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
new file mode 100644
index 0000000..3d92d68
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
@@ -0,0 +1,230 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFamilyProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
+
+public abstract class AbstractCompilerFactoryBuilder {
+
+    protected List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
+    protected List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
+    protected ITypeTraitProvider typeTraitProvider;
+    protected ISerializerDeserializerProvider serializerDeserializerProvider;
+    protected IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
+    protected IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider;
+    protected IBinaryComparatorFactoryProvider comparatorFactoryProvider;
+    protected IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
+    protected IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory;
+    protected IPrinterFactoryProvider printerProvider;
+    protected IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider;
+    protected IExpressionRuntimeProvider expressionRuntimeProvider;
+    protected IExpressionTypeComputer expressionTypeComputer;
+    protected INullableTypeComputer nullableTypeComputer;
+    protected IExpressionEvalSizeComputer expressionEvalSizeComputer;
+    protected INullWriterFactory nullWriterFactory;
+    protected INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
+    protected IPartialAggregationTypeComputer partialAggregationTypeComputer;
+    protected IMergeAggregationExpressionFactory mergeAggregationExpressionFactory;
+    protected PhysicalOptimizationConfig physicalOptimizationConfig = new PhysicalOptimizationConfig();
+    protected AlgebricksPartitionConstraint clusterLocations;
+
+    public abstract ICompilerFactory create();
+
+    public void setLogicalRewrites(List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites) {
+        this.logicalRewrites = logicalRewrites;
+    }
+
+    public void setPhysicalRewrites(List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites) {
+        this.physicalRewrites = physicalRewrites;
+    }
+
+    public void setTypeTraitProvider(ITypeTraitProvider typeTraitProvider) {
+        this.typeTraitProvider = typeTraitProvider;
+    }
+
+    public ITypeTraitProvider getTypeTraitProvider() {
+        return typeTraitProvider;
+    }
+
+    public void setSerializerDeserializerProvider(ISerializerDeserializerProvider serializerDeserializerProvider) {
+        this.serializerDeserializerProvider = serializerDeserializerProvider;
+    }
+
+    public ISerializerDeserializerProvider getSerializerDeserializerProvider() {
+        return serializerDeserializerProvider;
+    }
+
+    public void setHashFunctionFactoryProvider(IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider) {
+        this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
+    }
+
+    public IBinaryHashFunctionFactoryProvider getHashFunctionFactoryProvider() {
+        return hashFunctionFactoryProvider;
+    }
+
+    public void setHashFunctionFamilyProvider(IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider) {
+        this.hashFunctionFamilyProvider = hashFunctionFamilyProvider;
+    }
+
+    public IBinaryHashFunctionFamilyProvider getHashFunctionFamilyProvider() {
+        return hashFunctionFamilyProvider;
+    }
+
+    public void setComparatorFactoryProvider(IBinaryComparatorFactoryProvider comparatorFactoryProvider) {
+        this.comparatorFactoryProvider = comparatorFactoryProvider;
+    }
+
+    public IBinaryComparatorFactoryProvider getComparatorFactoryProvider() {
+        return comparatorFactoryProvider;
+    }
+    
+    public void setPredicateEvaluatorFactoryProvider(IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider) {
+        this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
+    }
+
+    public IPredicateEvaluatorFactoryProvider getPredicateEvaluatorFactory() {
+        return predEvaluatorFactoryProvider;
+    }
+
+    public void setBinaryBooleanInspectorFactory(IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
+        this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
+    }
+
+    public IBinaryBooleanInspectorFactory getBinaryBooleanInspectorFactory() {
+        return binaryBooleanInspectorFactory;
+    }
+
+    public void setBinaryIntegerInspectorFactory(IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory) {
+        this.binaryIntegerInspectorFactory = binaryIntegerInspectorFactory;
+    }
+
+    public IBinaryIntegerInspectorFactory getBinaryIntegerInspectorFactory() {
+        return binaryIntegerInspectorFactory;
+    }
+
+    public void setPrinterProvider(IPrinterFactoryProvider printerProvider) {
+        this.printerProvider = printerProvider;
+    }
+
+    public IPrinterFactoryProvider getPrinterProvider() {
+        return printerProvider;
+    }
+
+    public void setExpressionRuntimeProvider(IExpressionRuntimeProvider expressionRuntimeProvider) {
+        this.expressionRuntimeProvider = expressionRuntimeProvider;
+    }
+
+    public IExpressionRuntimeProvider getExpressionRuntimeProvider() {
+        return expressionRuntimeProvider;
+    }
+
+    public void setExpressionTypeComputer(IExpressionTypeComputer expressionTypeComputer) {
+        this.expressionTypeComputer = expressionTypeComputer;
+    }
+
+    public IExpressionTypeComputer getExpressionTypeComputer() {
+        return expressionTypeComputer;
+    }
+
+    public void setClusterLocations(AlgebricksPartitionConstraint clusterLocations) {
+        this.clusterLocations = clusterLocations;
+    }
+
+    public AlgebricksPartitionConstraint getClusterLocations() {
+        return clusterLocations;
+    }
+
+    public void setNullWriterFactory(INullWriterFactory nullWriterFactory) {
+        this.nullWriterFactory = nullWriterFactory;
+    }
+
+    public INullWriterFactory getNullWriterFactory() {
+        return nullWriterFactory;
+    }
+
+    public void setExpressionEvalSizeComputer(IExpressionEvalSizeComputer expressionEvalSizeComputer) {
+        this.expressionEvalSizeComputer = expressionEvalSizeComputer;
+    }
+
+    public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
+        return expressionEvalSizeComputer;
+    }
+
+    public void setNormalizedKeyComputerFactoryProvider(
+            INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider) {
+        this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
+    }
+
+    public INormalizedKeyComputerFactoryProvider getNormalizedKeyComputerFactoryProvider() {
+        return normalizedKeyComputerFactoryProvider;
+    }
+
+    public IPartialAggregationTypeComputer getPartialAggregationTypeComputer() {
+        return partialAggregationTypeComputer;
+    }
+
+    public void setPartialAggregationTypeComputer(IPartialAggregationTypeComputer partialAggregationTypeComputer) {
+        this.partialAggregationTypeComputer = partialAggregationTypeComputer;
+    }
+
+    public IMergeAggregationExpressionFactory getIMergeAggregationExpressionFactory() {
+        return mergeAggregationExpressionFactory;
+    }
+
+    public void setIMergeAggregationExpressionFactory(
+            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory) {
+        this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
+    }
+
+    public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
+        return physicalOptimizationConfig;
+    }
+
+    public void setPhysicalOptimizationConfig(PhysicalOptimizationConfig physicalOptimizationConfig) {
+        this.physicalOptimizationConfig = physicalOptimizationConfig;
+    }
+
+    public void setNullableTypeComputer(INullableTypeComputer nullableTypeComputer) {
+        this.nullableTypeComputer = nullableTypeComputer;
+    }
+
+    public INullableTypeComputer getNullableTypeComputer() {
+        return nullableTypeComputer;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
new file mode 100644
index 0000000..26dc343
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuilder {
+
+    public static class DefaultOptimizationContextFactory implements IOptimizationContextFactory {
+
+        public static final DefaultOptimizationContextFactory INSTANCE = new DefaultOptimizationContextFactory();
+
+        private DefaultOptimizationContextFactory() {
+        }
+
+        @Override
+        public IOptimizationContext createOptimizationContext(int varCounter,
+                IExpressionEvalSizeComputer expressionEvalSizeComputer,
+                IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+                IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+                PhysicalOptimizationConfig physicalOptimizationConfig) {
+            LogicalOperatorPrettyPrintVisitor prettyPrintVisitor = new LogicalOperatorPrettyPrintVisitor();
+            return new AlgebricksOptimizationContext(varCounter, expressionEvalSizeComputer,
+                    mergeAggregationExpressionFactory, expressionTypeComputer, nullableTypeComputer,
+                    physicalOptimizationConfig, prettyPrintVisitor);
+        }
+    }
+
+    private IOptimizationContextFactory optCtxFactory;
+
+    public HeuristicCompilerFactoryBuilder() {
+        this.optCtxFactory = DefaultOptimizationContextFactory.INSTANCE;
+    }
+
+    public HeuristicCompilerFactoryBuilder(IOptimizationContextFactory optCtxFactory) {
+        this.optCtxFactory = optCtxFactory;
+    }
+
+    @Override
+    public ICompilerFactory create() {
+        return new ICompilerFactory() {
+            @Override
+            public ICompiler createCompiler(final ILogicalPlan plan, final IMetadataProvider<?, ?> metadata,
+                    int varCounter) {
+                final IOptimizationContext oc = optCtxFactory.createOptimizationContext(varCounter,
+                        expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
+                        nullableTypeComputer, physicalOptimizationConfig);
+                oc.setMetadataDeclarations(metadata);
+                final HeuristicOptimizer opt = new HeuristicOptimizer(plan, logicalRewrites, physicalRewrites, oc);
+                return new ICompiler() {
+
+                    @Override
+                    public void optimize() throws AlgebricksException {
+                        opt.optimize();
+                    }
+
+                    @Override
+                    public JobSpecification createJob(Object appContext,
+                            IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException {
+                        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting Job Generation.\n");
+                        JobGenContext context = new JobGenContext(null, metadata, appContext,
+                                serializerDeserializerProvider, hashFunctionFactoryProvider,
+                                hashFunctionFamilyProvider, comparatorFactoryProvider, typeTraitProvider,
+                                binaryBooleanInspectorFactory, binaryIntegerInspectorFactory, printerProvider,
+                                nullWriterFactory, normalizedKeyComputerFactoryProvider, expressionRuntimeProvider,
+                                expressionTypeComputer, nullableTypeComputer, oc, expressionEvalSizeComputer,
+                                partialAggregationTypeComputer, predEvaluatorFactoryProvider,
+                                physicalOptimizationConfig.getFrameSize(), clusterLocations);
+
+                        PlanCompiler pc = new PlanCompiler(context);
+                        return pc.compilePlan(plan, null, jobEventListenerFactory);
+                    }
+                };
+            }
+        };
+    }
+
+}

[02/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
deleted file mode 100644
index b328aab..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public abstract class AbstractOneInputSinkPushRuntime implements IPushRuntime {
-    protected RecordDescriptor inputRecordDesc;
-
-    @Override
-    public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
-        throw new IllegalStateException();
-    }
-
-    @Override
-    public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
-        this.inputRecordDesc = recordDescriptor;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
deleted file mode 100644
index 8e71cd7..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public abstract class AbstractOneInputSourcePushRuntime extends AbstractOneInputPushRuntime {
-
-    @Override
-    public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public void close() throws HyracksDataException {
-    }
-
-    @Override
-    public void fail() throws HyracksDataException {
-        writer.fail();
-    }
-
-    @Override
-    public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
-        throw new UnsupportedOperationException();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java
deleted file mode 100644
index 33f28c3..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/base/SinkRuntimeFactory.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.base;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class SinkRuntimeFactory implements IPushRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    public SinkRuntimeFactory() {
-    }
-
-    @Override
-    public String toString() {
-        return "sink";
-    }
-
-    @Override
-    public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
-        return new AbstractOneInputSinkPushRuntime() {
-
-            @Override
-            public void open() throws HyracksDataException {
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-            }
-
-            @Override
-            public void close() throws HyracksDataException {
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
deleted file mode 100644
index 3380d08..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.group;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupWriter;
-
-public class MicroPreClusteredGroupRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-    private final int[] groupFields;
-    private final IBinaryComparatorFactory[] comparatorFactories;
-    private final IAggregatorDescriptorFactory aggregatorFactory;
-    private final RecordDescriptor inRecordDesc;
-    private final RecordDescriptor outRecordDesc;
-
-    public MicroPreClusteredGroupRuntimeFactory(int[] groupFields, IBinaryComparatorFactory[] comparatorFactories,
-            IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDesc,
-            RecordDescriptor outRecordDesc, int[] projectionList) {
-        super(projectionList);
-        // Obs: the projection list is currently ignored.
-        if (projectionList != null) {
-            throw new NotImplementedException("Cannot push projection into InMemorySortRuntime.");
-        }
-        this.groupFields = groupFields;
-        this.comparatorFactories = comparatorFactories;
-        this.aggregatorFactory = aggregatorFactory;
-        this.inRecordDesc = inRecordDesc;
-        this.outRecordDesc = outRecordDesc;
-    }
-
-    @Override
-    public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException {
-        final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
-        for (int i = 0; i < comparatorFactories.length; ++i) {
-            comparators[i] = comparatorFactories[i].createBinaryComparator();
-        }
-
-        return new AbstractOneInputOneOutputPushRuntime() {
-
-            private PreclusteredGroupWriter pgw;
-
-            @Override
-            public void open() throws HyracksDataException {
-                pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregatorFactory, inRecordDesc,
-                        outRecordDesc, writer);
-                pgw.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                pgw.nextFrame(buffer);
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-                pgw.fail();
-            }
-
-            @Override
-            public void close() throws HyracksDataException {
-                pgw.close();
-            }
-        };
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
deleted file mode 100644
index 06ef109..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.meta;
-
-import java.nio.ByteBuffer;
-
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
-
-public class AlgebricksMetaOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
-
-    private static final long serialVersionUID = 1L;
-
-    // array of factories for building the local runtime pipeline
-    private final AlgebricksPipeline pipeline;
-
-    public AlgebricksMetaOperatorDescriptor(IOperatorDescriptorRegistry spec, int inputArity, int outputArity,
-            IPushRuntimeFactory[] runtimeFactories, RecordDescriptor[] internalRecordDescriptors) {
-        super(spec, inputArity, outputArity);
-        if (outputArity == 1) {
-            this.recordDescriptors[0] = internalRecordDescriptors[internalRecordDescriptors.length - 1];
-        }
-        this.pipeline = new AlgebricksPipeline(runtimeFactories, internalRecordDescriptors);
-    }
-
-    public AlgebricksPipeline getPipeline() {
-        return pipeline;
-    }
-
-    @Override
-    public JSONObject toJSON() throws JSONException {
-        JSONObject json = super.toJSON();
-        json.put("micro-operators", pipeline.getRuntimeFactories());
-        return json;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("Asterix { \n");
-        for (IPushRuntimeFactory f : pipeline.getRuntimeFactories()) {
-            sb.append("  " + f.toString() + ";\n");
-        }
-        sb.append("}");
-        // sb.append(super.getInputArity());
-        // sb.append(";");
-        // sb.append(super.getOutputArity());
-        // sb.append(";");
-        return sb.toString();
-    }
-
-    @Override
-    public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
-            final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
-        if (inputArity == 0) {
-            return createSourceInputPushRuntime(ctx, recordDescProvider, partition, nPartitions);
-        } else {
-            return createOneInputOneOutputPushRuntime(ctx, recordDescProvider, partition, nPartitions);
-        }
-    }
-
-    private IOperatorNodePushable createSourceInputPushRuntime(final IHyracksTaskContext ctx,
-            final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
-        return new AbstractUnaryOutputSourceOperatorNodePushable() {
-
-            public void initialize() throws HyracksDataException {
-                IFrameWriter startOfPipeline;
-                RecordDescriptor pipelineOutputRecordDescriptor = outputArity > 0 ? AlgebricksMetaOperatorDescriptor.this.recordDescriptors[0]
-                        : null;
-
-                PipelineAssembler pa = new PipelineAssembler(pipeline, inputArity, outputArity, null,
-                        pipelineOutputRecordDescriptor);
-                try {
-                    startOfPipeline = pa.assemblePipeline(writer, ctx);
-                } catch (AlgebricksException e) {
-                    throw new HyracksDataException(e);
-                }
-                startOfPipeline.open();
-                startOfPipeline.close();
-            }
-        };
-    }
-
-    private IOperatorNodePushable createOneInputOneOutputPushRuntime(final IHyracksTaskContext ctx,
-            final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
-        return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
-
-            private IFrameWriter startOfPipeline;
-
-            @Override
-            public void open() throws HyracksDataException {
-                if (startOfPipeline == null) {
-                    RecordDescriptor pipelineOutputRecordDescriptor = outputArity > 0 ? AlgebricksMetaOperatorDescriptor.this.recordDescriptors[0]
-                            : null;
-                    RecordDescriptor pipelineInputRecordDescriptor = recordDescProvider.getInputRecordDescriptor(
-                            AlgebricksMetaOperatorDescriptor.this.getActivityId(), 0);
-                    PipelineAssembler pa = new PipelineAssembler(pipeline, inputArity, outputArity,
-                            pipelineInputRecordDescriptor, pipelineOutputRecordDescriptor);
-                    try {
-                        startOfPipeline = pa.assemblePipeline(writer, ctx);
-                    } catch (AlgebricksException ae) {
-                        throw new HyracksDataException(ae);
-                    }
-                }
-                startOfPipeline.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                startOfPipeline.nextFrame(buffer);
-            }
-
-            @Override
-            public void close() throws HyracksDataException {
-                startOfPipeline.close();
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-                startOfPipeline.fail();
-            }
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java
deleted file mode 100644
index 5bcf933..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/PipelineAssembler.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.meta;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class PipelineAssembler {
-
-    // array of factories for building the local runtime pipeline
-    private final RecordDescriptor pipelineInputRecordDescriptor;
-    private final RecordDescriptor pipelineOutputRecordDescriptor;
-
-    private final int inputArity;
-    private final int outputArity;
-    private final AlgebricksPipeline pipeline;
-
-    public PipelineAssembler(AlgebricksPipeline pipeline, int inputArity, int outputArity,
-            RecordDescriptor pipelineInputRecordDescriptor, RecordDescriptor pipelineOutputRecordDescriptor) {
-        this.pipeline = pipeline;
-        this.pipelineInputRecordDescriptor = pipelineInputRecordDescriptor;
-        this.pipelineOutputRecordDescriptor = pipelineOutputRecordDescriptor;
-        this.inputArity = inputArity;
-        this.outputArity = outputArity;
-    }
-
-    public IFrameWriter assemblePipeline(IFrameWriter writer, IHyracksTaskContext ctx) throws AlgebricksException,
-            HyracksDataException {
-        // plug the operators
-        IFrameWriter start = writer;// this.writer;
-        for (int i = pipeline.getRuntimeFactories().length - 1; i >= 0; i--) {
-            IPushRuntime newRuntime = pipeline.getRuntimeFactories()[i].createPushRuntime(ctx);
-            if (i == pipeline.getRuntimeFactories().length - 1) {
-                if (outputArity == 1) {
-                    newRuntime.setFrameWriter(0, start, pipelineOutputRecordDescriptor);
-                }
-            } else {
-                newRuntime.setFrameWriter(0, start, pipeline.getRecordDescriptors()[i]);
-            }
-            if (i > 0) {
-                newRuntime.setInputRecordDescriptor(0, pipeline.getRecordDescriptors()[i - 1]);
-            } else if (inputArity > 0) {
-                newRuntime.setInputRecordDescriptor(0, pipelineInputRecordDescriptor);
-            }
-            start = newRuntime;
-        }
-        return start;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java
deleted file mode 100644
index cf40669..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/meta/SubplanRuntimeFactory.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.meta;
-
-import java.io.DataOutput;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-
-public class SubplanRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final AlgebricksPipeline pipeline;
-    private final RecordDescriptor inputRecordDesc;
-    private final INullWriterFactory[] nullWriterFactories;
-
-    public SubplanRuntimeFactory(AlgebricksPipeline pipeline, INullWriterFactory[] nullWriterFactories,
-            RecordDescriptor inputRecordDesc, int[] projectionList) {
-        super(projectionList);
-        this.pipeline = pipeline;
-        this.nullWriterFactories = nullWriterFactories;
-        this.inputRecordDesc = inputRecordDesc;
-        if (projectionList != null) {
-            throw new NotImplementedException();
-        }
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("Subplan { \n");
-        for (IPushRuntimeFactory f : pipeline.getRuntimeFactories()) {
-            sb.append("  " + f.toString() + ";\n");
-        }
-        sb.append("}");
-        return sb.toString();
-    }
-
-    @Override
-    public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException, HyracksDataException {
-
-        RecordDescriptor pipelineOutputRecordDescriptor = null;
-
-        final PipelineAssembler pa = new PipelineAssembler(pipeline, 1, 1, inputRecordDesc,
-                pipelineOutputRecordDescriptor);
-        final INullWriter[] nullWriters = new INullWriter[nullWriterFactories.length];
-        for (int i = 0; i < nullWriterFactories.length; i++) {
-            nullWriters[i] = nullWriterFactories[i].createNullWriter();
-        }
-
-        return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
-            /**
-             * Computes the outer product between a given tuple and the frames
-             * passed.
-             */
-            class TupleOuterProduct implements IFrameWriter {
-
-                private boolean smthWasWritten = false;
-                private FrameTupleAccessor ta = new FrameTupleAccessor(
-                        pipeline.getRecordDescriptors()[pipeline.getRecordDescriptors().length - 1]);
-                private ArrayTupleBuilder tb = new ArrayTupleBuilder(nullWriters.length);
-
-                @Override
-                public void open() throws HyracksDataException {
-                    smthWasWritten = false;
-                }
-
-                @Override
-                public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                    ta.reset(buffer);
-                    int nTuple = ta.getTupleCount();
-                    for (int t = 0; t < nTuple; t++) {
-                        appendConcat(tRef.getFrameTupleAccessor(), tRef.getTupleIndex(), ta, t);
-                    }
-                    smthWasWritten = true;
-                }
-
-                @Override
-                public void close() throws HyracksDataException {
-                    if (!smthWasWritten) {
-                        // the case when we need to write nulls
-                        appendNullsToTuple();
-                        appendToFrameFromTupleBuilder(tb);
-                    }
-                }
-
-                @Override
-                public void fail() throws HyracksDataException {
-                    writer.fail();
-                }
-
-                private void appendNullsToTuple() throws HyracksDataException {
-                    tb.reset();
-                    int n0 = tRef.getFieldCount();
-                    for (int f = 0; f < n0; f++) {
-                        tb.addField(tRef.getFrameTupleAccessor(), tRef.getTupleIndex(), f);
-                    }
-                    DataOutput dos = tb.getDataOutput();
-                    for (int i = 0; i < nullWriters.length; i++) {
-                        nullWriters[i].writeNull(dos);
-                        tb.addFieldEndOffset();
-                    }
-                }
-
-            }
-
-            IFrameWriter endPipe = new TupleOuterProduct();
-
-            NestedTupleSourceRuntime startOfPipeline = (NestedTupleSourceRuntime) pa.assemblePipeline(endPipe, ctx);
-
-            boolean first = true;
-
-            @Override
-            public void open() throws HyracksDataException {
-                if (first) {
-                    first = false;
-                    initAccessAppendRef(ctx);
-                }
-                writer.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-                for (int t = 0; t < nTuple; t++) {
-                    tRef.reset(tAccess, t);
-                    startOfPipeline.writeTuple(buffer, t);
-                    startOfPipeline.open();
-                    startOfPipeline.close();
-                }
-            }
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java
deleted file mode 100644
index d3751f5..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/sort/InMemorySortRuntimeFactory.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.sort;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.std.sort.FrameSorterMergeSort;
-import edu.uci.ics.hyracks.dataflow.std.sort.buffermanager.FrameFreeSlotLastFit;
-import edu.uci.ics.hyracks.dataflow.std.sort.buffermanager.IFrameBufferManager;
-import edu.uci.ics.hyracks.dataflow.std.sort.buffermanager.VariableFramePool;
-import edu.uci.ics.hyracks.dataflow.std.sort.buffermanager.VariableFrameMemoryManager;
-
-public class InMemorySortRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final int[] sortFields;
-    private INormalizedKeyComputerFactory firstKeyNormalizerFactory;
-    private IBinaryComparatorFactory[] comparatorFactories;
-
-    public InMemorySortRuntimeFactory(int[] sortFields, INormalizedKeyComputerFactory firstKeyNormalizerFactory,
-            IBinaryComparatorFactory[] comparatorFactories, int[] projectionList) {
-        super(projectionList);
-        // Obs: the projection list is currently ignored.
-        if (projectionList != null) {
-            throw new NotImplementedException("Cannot push projection into InMemorySortRuntime.");
-        }
-        this.sortFields = sortFields;
-        this.firstKeyNormalizerFactory = firstKeyNormalizerFactory;
-        this.comparatorFactories = comparatorFactories;
-    }
-
-    @Override
-    public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException {
-
-        return new AbstractOneInputOneOutputPushRuntime() {
-
-            FrameSorterMergeSort frameSorter = null;
-
-            @Override
-            public void open() throws HyracksDataException {
-                if (frameSorter == null) {
-                    IFrameBufferManager manager = new VariableFrameMemoryManager(
-                            new VariableFramePool(ctx, VariableFramePool.UNLIMITED_MEMORY),
-                            new FrameFreeSlotLastFit());
-                    frameSorter = new FrameSorterMergeSort(ctx, manager, sortFields, firstKeyNormalizerFactory,
-                            comparatorFactories, outputRecordDesc);
-                }
-                frameSorter.reset();
-                writer.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                frameSorter.insertFrame(buffer);
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-                writer.fail();
-            }
-
-            @Override
-            public void close() throws HyracksDataException {
-                frameSorter.sort();
-                frameSorter.flush(writer);
-                writer.close();
-            }
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
deleted file mode 100644
index fb889ea..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/AssignRuntimeFactory.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-
-public class AssignRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private int[] outColumns;
-    private IScalarEvaluatorFactory[] evalFactories;
-    private final boolean flushFramesRapidly;
-
-    /**
-     * @param outColumns
-     *            a sorted array of columns into which the result is written to
-     * @param evalFactories
-     * @param projectionList
-     *            an array of columns to be projected
-     */
-
-    public AssignRuntimeFactory(int[] outColumns, IScalarEvaluatorFactory[] evalFactories, int[] projectionList) {
-        this(outColumns, evalFactories, projectionList, false);
-    }
-
-    public AssignRuntimeFactory(int[] outColumns, IScalarEvaluatorFactory[] evalFactories, int[] projectionList,
-            boolean flushFramesRapidly) {
-        super(projectionList);
-        this.outColumns = outColumns;
-        this.evalFactories = evalFactories;
-        this.flushFramesRapidly = flushFramesRapidly;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("assign [");
-        for (int i = 0; i < outColumns.length; i++) {
-            if (i > 0) {
-                sb.append(", ");
-            }
-            sb.append(outColumns[i]);
-        }
-        sb.append("] := [");
-        for (int i = 0; i < evalFactories.length; i++) {
-            if (i > 0) {
-                sb.append(", ");
-            }
-            sb.append(evalFactories[i]);
-        }
-        sb.append("]");
-        return sb.toString();
-    }
-
-    @Override
-    public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException {
-        final int[] projectionToOutColumns = new int[projectionList.length];
-        for (int j = 0; j < projectionList.length; j++) {
-            projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
-        }
-
-        return new AbstractOneInputOneOutputOneFramePushRuntime() {
-            private IPointable result = VoidPointable.FACTORY.createPointable();
-            private IScalarEvaluator[] eval = new IScalarEvaluator[evalFactories.length];
-            private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);
-            private boolean first = true;
-
-            @Override
-            public void open() throws HyracksDataException {
-                if (first) {
-                    initAccessAppendRef(ctx);
-                    first = false;
-                    int n = evalFactories.length;
-                    for (int i = 0; i < n; i++) {
-                        try {
-                            eval[i] = evalFactories[i].createScalarEvaluator(ctx);
-                        } catch (AlgebricksException ae) {
-                            throw new HyracksDataException(ae);
-                        }
-                    }
-                }
-                writer.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-                int t = 0;
-                if (nTuple > 1) {
-                    for (; t < nTuple - 1; t++) {
-                        tRef.reset(tAccess, t);
-                        produceTuple(tupleBuilder, tAccess, t, tRef);
-                        appendToFrameFromTupleBuilder(tupleBuilder);
-                    }
-                }
-
-                tRef.reset(tAccess, t);
-                produceTuple(tupleBuilder, tAccess, t, tRef);
-                if (flushFramesRapidly) {
-                    // Whenever all the tuples in the incoming frame have been consumed, the assign operator 
-                    // will push its frame to the next operator; i.e., it won't wait until the frame gets full. 
-                    appendToFrameFromTupleBuilder(tupleBuilder, true);
-                } else {
-                    appendToFrameFromTupleBuilder(tupleBuilder);
-                }
-            }
-
-            private void produceTuple(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
-                    FrameTupleReference tupleRef) throws HyracksDataException {
-                tb.reset();
-                for (int f = 0; f < projectionList.length; f++) {
-                    int k = projectionToOutColumns[f];
-                    if (k >= 0) {
-                        try {
-                            eval[k].evaluate(tupleRef, result);
-                        } catch (AlgebricksException e) {
-                            throw new HyracksDataException(e);
-                        }
-                        tb.addField(result.getByteArray(), result.getStartOffset(), result.getLength());
-                    } else {
-                        tb.addField(accessor, tIndex, projectionList[f]);
-                    }
-                }
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-                writer.fail();
-            }
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
deleted file mode 100644
index 35fcafc..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputSourcePushRuntime;
-import edu.uci.ics.hyracks.api.comm.VSizeFrame;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-
-public class EmptyTupleSourceRuntimeFactory implements IPushRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    public EmptyTupleSourceRuntimeFactory() {
-    }
-
-    @Override
-    public String toString() {
-        return "ets";
-    }
-
-    @Override
-    public IPushRuntime createPushRuntime(final IHyracksTaskContext ctx) throws HyracksDataException {
-        return new AbstractOneInputSourcePushRuntime() {
-
-            private ArrayTupleBuilder tb = new ArrayTupleBuilder(0);
-            private FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
-
-            @Override
-            public void open() throws HyracksDataException {
-                writer.open();
-                if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
-                    throw new IllegalStateException();
-                }
-                appender.flush(writer, true);
-                writer.close();
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
deleted file mode 100644
index 8df87ab..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class NestedTupleSourceRuntimeFactory implements IPushRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    public NestedTupleSourceRuntimeFactory() {
-    }
-
-    @Override
-    public String toString() {
-        return "nts";
-    }
-
-    @Override
-    public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws HyracksDataException {
-        return new NestedTupleSourceRuntime(ctx);
-    }
-
-    public static class NestedTupleSourceRuntime extends AbstractOneInputOneOutputOneFramePushRuntime {
-
-        public NestedTupleSourceRuntime(IHyracksTaskContext ctx) throws HyracksDataException {
-            initAccessAppend(ctx);
-        }
-
-        @Override
-        public void open() throws HyracksDataException {
-            writer.open();
-        }
-
-        public void writeTuple(ByteBuffer inputBuffer, int tIndex) throws HyracksDataException {
-            tAccess.reset(inputBuffer);
-            appendTupleToFrame(tIndex);
-        }
-
-        @Override
-        public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-            throw new IllegalStateException();
-        }
-
-        @Override
-        public void fail() throws HyracksDataException {
-            writer.fail();
-        }
-
-        public void forceFlush() throws HyracksDataException {
-            appender.flush(writer, true);
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java
deleted file mode 100644
index 1d2d27c..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PartitioningSplitOperatorDescriptor.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.api.comm.IFrame;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.comm.VSizeFrame;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputOperatorNodePushable;
-
-public class PartitioningSplitOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
-    private static final long serialVersionUID = 1L;
-    public static int NO_DEFAULT_BRANCH = -1;
-
-    private final ICopyEvaluatorFactory[] evalFactories;
-    private final IBinaryBooleanInspector boolInspector;
-    private final int defaultBranchIndex;
-
-    public PartitioningSplitOperatorDescriptor(IOperatorDescriptorRegistry spec, ICopyEvaluatorFactory[] evalFactories,
-            IBinaryBooleanInspector boolInspector, int defaultBranchIndex, RecordDescriptor rDesc) {
-        super(spec, 1, (defaultBranchIndex == evalFactories.length) ? evalFactories.length + 1 : evalFactories.length);
-        for (int i = 0; i < evalFactories.length; i++) {
-            recordDescriptors[i] = rDesc;
-        }
-        this.evalFactories = evalFactories;
-        this.boolInspector = boolInspector;
-        this.defaultBranchIndex = defaultBranchIndex;
-    }
-
-    @Override
-    public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
-            final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
-            throws HyracksDataException {
-        return new AbstractUnaryInputOperatorNodePushable() {
-            private final IFrameWriter[] writers = new IFrameWriter[outputArity];
-            private final IFrame[] writeBuffers = new IFrame[outputArity];
-            private final ICopyEvaluator[] evals = new ICopyEvaluator[outputArity];
-            private final ArrayBackedValueStorage evalBuf = new ArrayBackedValueStorage();
-            private final RecordDescriptor inOutRecDesc = recordDescProvider.getInputRecordDescriptor(getActivityId(),
-                    0);
-            private final FrameTupleAccessor accessor = new FrameTupleAccessor(inOutRecDesc);
-            private final FrameTupleReference frameTuple = new FrameTupleReference();
-
-            private final FrameTupleAppender tupleAppender = new FrameTupleAppender();
-            private final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(inOutRecDesc.getFieldCount());
-            private final DataOutput tupleDos = tupleBuilder.getDataOutput();
-
-            @Override
-            public void close() throws HyracksDataException {
-                // Flush (possibly not full) buffers that have data, and close writers.
-                for (int i = 0; i < outputArity; i++) {
-                    tupleAppender.reset(writeBuffers[i], false);
-                    // ? by JF why didn't clear the buffer ?
-                    tupleAppender.flush(writers[i], false);
-                    writers[i].close();
-                }
-            }
-
-            @Override
-            public void fail() throws HyracksDataException {
-                for (IFrameWriter writer : writers) {
-                    writer.fail();
-                }
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                accessor.reset(buffer);
-                int tupleCount = accessor.getTupleCount();
-                for (int i = 0; i < tupleCount; i++) {
-                    frameTuple.reset(accessor, i);
-                    boolean found = false;
-                    for (int j = 0; j < evals.length; j++) {
-                        try {
-                            evalBuf.reset();
-                            evals[j].evaluate(frameTuple);
-                        } catch (AlgebricksException e) {
-                            throw new HyracksDataException(e);
-                        }
-                        found = boolInspector.getBooleanValue(evalBuf.getByteArray(), 0, 1);
-                        if (found) {
-                            copyAndAppendTuple(j);
-                            break;
-                        }
-                    }
-                    // Optionally write to default output branch.
-                    if (!found && defaultBranchIndex != NO_DEFAULT_BRANCH) {
-                        copyAndAppendTuple(defaultBranchIndex);
-                    }
-                }
-            }
-
-            private void copyAndAppendTuple(int outputIndex) throws HyracksDataException {
-                // Copy tuple into tuple builder.
-                try {
-                    tupleBuilder.reset();
-                    for (int i = 0; i < frameTuple.getFieldCount(); i++) {
-                        tupleDos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
-                                frameTuple.getFieldLength(i));
-                        tupleBuilder.addFieldEndOffset();
-                    }
-                } catch (IOException e) {
-                    throw new HyracksDataException(e);
-                }
-                FrameUtils.appendToWriter(writers[outputIndex], tupleAppender, tupleBuilder.getFieldEndOffsets(),
-                        tupleBuilder.getByteArray(), 0, tupleBuilder.getSize());
-            }
-
-            @Override
-            public void open() throws HyracksDataException {
-                for (IFrameWriter writer : writers) {
-                    writer.open();
-                }
-                // Create write buffers.
-                for (int i = 0; i < outputArity; i++) {
-                    writeBuffers[i] = new VSizeFrame(ctx);
-                    // Make sure to clear all buffers, since we are reusing the tupleAppender.
-                    tupleAppender.reset(writeBuffers[i], true);
-                }
-                // Create evaluators for partitioning.
-                try {
-                    for (int i = 0; i < evalFactories.length; i++) {
-                        evals[i] = evalFactories[i].createEvaluator(evalBuf);
-                    }
-                } catch (AlgebricksException e) {
-                    throw new HyracksDataException(e);
-                }
-            }
-
-            @Override
-            public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
-                writers[index] = writer;
-            }
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java
deleted file mode 100644
index c2ecc56..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/PrinterRuntimeFactory.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import edu.uci.ics.hyracks.algebricks.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class PrinterRuntimeFactory implements IPushRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final int[] printColumns;
-    private final IPrinterFactory[] printerFactories;
-    private final RecordDescriptor inputRecordDesc;
-
-    public PrinterRuntimeFactory(int[] printColumns, IPrinterFactory[] printerFactories,
-            RecordDescriptor inputRecordDesc) {
-        this.printColumns = printColumns;
-        this.printerFactories = printerFactories;
-        this.inputRecordDesc = inputRecordDesc;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder buf = new StringBuilder();
-        buf.append("print [");
-        for (int i = 0; i < printColumns.length; i++) {
-            if (i > 0) {
-                buf.append("; ");
-            }
-            buf.append(printColumns[i]);
-        }
-        buf.append("]");
-        return buf.toString();
-    }
-
-    @Override
-    public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) {
-        IAWriter w = PrinterBasedWriterFactory.INSTANCE.createWriter(printColumns, System.out, printerFactories,
-                inputRecordDesc);
-        return new SinkWriterRuntime(w, ctx, System.out, inputRecordDesc);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java
deleted file mode 100644
index 154f2d1..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/RunningAggregateRuntimeFactory.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-
-public class RunningAggregateRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final int[] outColumns;
-    private final IRunningAggregateEvaluatorFactory[] runningAggregates;
-
-    /**
-     * @param outColumns
-     *            a sorted array of columns into which the result is written to
-     * @param runningAggregates
-     * @param projectionList
-     *            an array of columns to be projected
-     */
-
-    public RunningAggregateRuntimeFactory(int[] outColumns, IRunningAggregateEvaluatorFactory[] runningAggregates,
-            int[] projectionList) {
-        super(projectionList);
-        this.outColumns = outColumns;
-        this.runningAggregates = runningAggregates;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("running-aggregate [");
-        for (int i = 0; i < outColumns.length; i++) {
-            if (i > 0) {
-                sb.append(", ");
-            }
-            sb.append(outColumns[i]);
-        }
-        sb.append("] := [");
-        for (int i = 0; i < runningAggregates.length; i++) {
-            if (i > 0) {
-                sb.append(", ");
-            }
-            sb.append(runningAggregates[i]);
-        }
-        sb.append("]");
-        return sb.toString();
-    }
-
-    @Override
-    public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException {
-        final int[] projectionToOutColumns = new int[projectionList.length];
-        for (int j = 0; j < projectionList.length; j++) {
-            projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
-        }
-
-        return new AbstractOneInputOneOutputOneFramePushRuntime() {
-            private final IPointable p = VoidPointable.FACTORY.createPointable();
-            private final IRunningAggregateEvaluator[] raggs = new IRunningAggregateEvaluator[runningAggregates.length];
-            private final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);
-            private boolean first = true;
-
-            @Override
-            public void open() throws HyracksDataException {
-                initAccessAppendRef(ctx);
-                if (first) {
-                    first = false;
-                    int n = runningAggregates.length;
-                    for (int i = 0; i < n; i++) {
-                        try {
-                            raggs[i] = runningAggregates[i].createRunningAggregateEvaluator();
-                        } catch (AlgebricksException ae) {
-                            throw new HyracksDataException(ae);
-                        }
-                    }
-                }
-                for (int i = 0; i < runningAggregates.length; i++) {
-                    try {
-                        raggs[i].init();
-                    } catch (AlgebricksException ae) {
-                        throw new HyracksDataException(ae);
-                    }
-                }
-                writer.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-                for (int t = 0; t < nTuple; t++) {
-                    tRef.reset(tAccess, t);
-                    produceTuple(tupleBuilder, tAccess, t, tRef);
-                    appendToFrameFromTupleBuilder(tupleBuilder);
-                }
-            }
-
-            private void produceTuple(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
-                    FrameTupleReference tupleRef) throws HyracksDataException {
-                tb.reset();
-                for (int f = 0; f < projectionList.length; f++) {
-                    int k = projectionToOutColumns[f];
-                    if (k >= 0) {
-                        try {
-                            raggs[k].step(tupleRef, p);
-                        } catch (AlgebricksException e) {
-                            throw new HyracksDataException(e);
-                        }
-                        tb.addField(p.getByteArray(), p.getStartOffset(), p.getLength());
-                    } else {
-                        tb.addField(accessor, tIndex, projectionList[f]);
-                    }
-                }
-            }
-
-        };
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java
deleted file mode 100644
index d26d090..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntime.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.io.PrintStream;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputSinkPushRuntime;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-
-public class SinkWriterRuntime extends AbstractOneInputSinkPushRuntime {
-
-    private final IHyracksTaskContext ctx;
-    private final PrintStream printStream;
-    private final IAWriter writer;
-    private RecordDescriptor inputRecordDesc;
-    private FrameTupleAccessor tAccess;
-    private boolean autoClose = false;
-    private boolean first = true;
-
-    public SinkWriterRuntime(IAWriter writer, IHyracksTaskContext ctx, PrintStream printStream,
-            RecordDescriptor inputRecordDesc) {
-        this.writer = writer;
-        this.ctx = ctx;
-        this.printStream = printStream;
-        this.inputRecordDesc = inputRecordDesc;
-        this.tAccess = new FrameTupleAccessor(inputRecordDesc);
-    }
-
-    public SinkWriterRuntime(IAWriter writer, IHyracksTaskContext ctx, PrintStream printStream,
-            RecordDescriptor inputRecordDesc, boolean autoClose) {
-        this(writer, ctx, printStream, inputRecordDesc);
-        this.autoClose = autoClose;
-    }
-
-    @Override
-    public void open() throws HyracksDataException {
-        if (first) {
-            first = false;
-            tAccess = new FrameTupleAccessor(inputRecordDesc);
-            try {
-                writer.init();
-            } catch (AlgebricksException e) {
-                throw new HyracksDataException(e);
-            }
-        }
-    }
-
-    @Override
-    public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-        tAccess.reset(buffer);
-        int nTuple = tAccess.getTupleCount();
-        for (int t = 0; t < nTuple; t++) {
-            try {
-                writer.printTuple(tAccess, t);
-            } catch (AlgebricksException ae) {
-                throw new HyracksDataException(ae);
-            }
-        }
-    }
-
-    @Override
-    public void close() throws HyracksDataException {
-        if (autoClose) {
-            printStream.close();
-        }
-    }
-
-    @Override
-    public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
-        this.inputRecordDesc = recordDescriptor;
-    }
-
-    @Override
-    public void fail() throws HyracksDataException {
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java
deleted file mode 100644
index a98bef3..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/SinkWriterRuntimeFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.PrintStream;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IAWriter;
-import edu.uci.ics.hyracks.algebricks.data.IAWriterFactory;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class SinkWriterRuntimeFactory implements IPushRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final int[] fields;
-    private final IPrinterFactory[] printerFactories;
-    private final File outputFile;
-    private final RecordDescriptor inputRecordDesc;
-    private final IAWriterFactory writerFactory;
-
-    public SinkWriterRuntimeFactory(int[] fields, IPrinterFactory[] printerFactories, File outputFile,
-            IAWriterFactory writerFactory, RecordDescriptor inputRecordDesc) {
-        this.fields = fields;
-        this.printerFactories = printerFactories;
-        this.outputFile = outputFile;
-        this.writerFactory = writerFactory;
-        this.inputRecordDesc = inputRecordDesc;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder buf = new StringBuilder();
-        buf.append("sink-write " + "[");
-        for (int i = 0; i < fields.length; i++) {
-            if (i > 0) {
-                buf.append("; ");
-            }
-            buf.append(fields[i]);
-        }
-        buf.append("] outputFile");
-        return buf.toString();
-    }
-
-    @Override
-    public IPushRuntime createPushRuntime(IHyracksTaskContext ctx) throws AlgebricksException {
-        PrintStream filePrintStream = null;
-        try {
-            filePrintStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile)));
-        } catch (FileNotFoundException e) {
-            throw new AlgebricksException(e);
-        }
-        IAWriter w = writerFactory.createWriter(fields, filePrintStream, printerFactories, inputRecordDesc);
-        return new SinkWriterRuntime(w, ctx, filePrintStream, inputRecordDesc, true);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
deleted file mode 100644
index 7d128aa..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamLimitRuntimeFactory.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspector;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-
-public class StreamLimitRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    private final IScalarEvaluatorFactory maxObjectsEvalFactory;
-    private final IScalarEvaluatorFactory offsetEvalFactory;
-    private final IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory;
-
-    public StreamLimitRuntimeFactory(IScalarEvaluatorFactory maxObjectsEvalFactory,
-            IScalarEvaluatorFactory offsetEvalFactory, int[] projectionList,
-            IBinaryIntegerInspectorFactory binaryIntegerInspectorFactory) {
-        super(projectionList);
-        this.maxObjectsEvalFactory = maxObjectsEvalFactory;
-        this.offsetEvalFactory = offsetEvalFactory;
-        this.binaryIntegerInspectorFactory = binaryIntegerInspectorFactory;
-    }
-
-    @Override
-    public String toString() {
-        String s = "stream-limit " + maxObjectsEvalFactory.toString();
-        if (offsetEvalFactory != null) {
-            return s + ", " + offsetEvalFactory.toString();
-        } else {
-            return s;
-        }
-    }
-
-    @Override
-    public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx) {
-        final IBinaryIntegerInspector bii = binaryIntegerInspectorFactory.createBinaryIntegerInspector(ctx);
-        return new AbstractOneInputOneOutputOneFramePushRuntime() {
-            private final IPointable p = VoidPointable.FACTORY.createPointable();
-            private IScalarEvaluator evalMaxObjects;
-            private IScalarEvaluator evalOffset = null;
-            private int toWrite = 0; // how many tuples still to write
-            private int toSkip = 0; // how many tuples still to skip
-            private boolean firstTuple = true;
-            private boolean afterLastTuple = false;
-
-            @Override
-            public void open() throws HyracksDataException {
-                // if (first) {
-                if (evalMaxObjects == null) {
-                    initAccessAppendRef(ctx);
-                    try {
-                        evalMaxObjects = maxObjectsEvalFactory.createScalarEvaluator(ctx);
-                        if (offsetEvalFactory != null) {
-                            evalOffset = offsetEvalFactory.createScalarEvaluator(ctx);
-                        }
-                    } catch (AlgebricksException ae) {
-                        throw new HyracksDataException(ae);
-                    }
-                }
-                writer.open();
-                afterLastTuple = false;
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                if (afterLastTuple) {
-                    // ignore the data
-                    return;
-                }
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-                int start = 0;
-                if (nTuple <= toSkip) {
-                    toSkip -= nTuple;
-                    return;
-                } else if (toSkip > 0) {
-                    start = toSkip;
-                    toSkip = 0;
-                }
-                for (int t = start; t < nTuple; t++) {
-                    if (firstTuple) {
-                        firstTuple = false;
-                        toWrite = evaluateInteger(evalMaxObjects, t);
-                        if (evalOffset != null) {
-                            toSkip = evaluateInteger(evalOffset, t);
-                        }
-                    }
-                    if (toSkip > 0) {
-                        toSkip--;
-                    } else if (toWrite > 0) {
-                        toWrite--;
-                        if (projectionList != null) {
-                            appendProjectionToFrame(t, projectionList);
-                        } else {
-                            appendTupleToFrame(t);
-                        }
-                    } else {
-                        // close();
-                        afterLastTuple = true;
-                        break;
-                    }
-                }
-            }
-
-            @Override
-            public void close() throws HyracksDataException {
-                toWrite = 0; // how many tuples still to write
-                toSkip = 0; // how many tuples still to skip
-                firstTuple = true;
-                afterLastTuple = false;
-                // if (!afterLastTuple) {
-                super.close();
-                // }
-            }
-
-            private int evaluateInteger(IScalarEvaluator eval, int tIdx) throws HyracksDataException {
-                tRef.reset(tAccess, tIdx);
-                try {
-                    eval.evaluate(tRef, p);
-                } catch (AlgebricksException ae) {
-                    throw new HyracksDataException(ae);
-                }
-                int lim = bii.getIntegerValue(p.getByteArray(), p.getStartOffset(), p.getLength());
-                return lim;
-            }
-
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java
deleted file mode 100644
index c0ae029..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/operators/std/StreamProjectRuntimeFactory.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.operators.std;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class StreamProjectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
-
-    private static final long serialVersionUID = 1L;
-    private final boolean flushFramesRapidly;
-
-    public StreamProjectRuntimeFactory(int[] projectionList, boolean flushFramesRapidly) {
-        super(projectionList);
-        this.flushFramesRapidly = flushFramesRapidly;
-    }
-
-    public StreamProjectRuntimeFactory(int[] projectionList) {
-        this(projectionList, false);
-    }
-
-    @Override
-    public String toString() {
-        return "stream-project " + Arrays.toString(projectionList);
-    }
-
-    @Override
-    public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final IHyracksTaskContext ctx)
-            throws AlgebricksException {
-
-        return new AbstractOneInputOneOutputOneFramePushRuntime() {
-
-            private boolean first = true;
-
-            @Override
-            public void open() throws HyracksDataException {
-                if (first) {
-                    first = false;
-                    initAccessAppend(ctx);
-                }
-                writer.open();
-            }
-
-            @Override
-            public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                tAccess.reset(buffer);
-                int nTuple = tAccess.getTupleCount();
-
-                int t = 0;
-                if (nTuple > 1) {
-                    for (; t < nTuple - 1; t++) {
-                        appendProjectionToFrame(t, projectionList);
-                    }
-                }
-                if (flushFramesRapidly) {
-                    // Whenever all the tuples in the incoming frame have been consumed, the project operator
-                    // will push its frame to the next operator; i.e., it won't wait until the frame gets full.
-                    appendProjectionToFrame(t, projectionList, true);
-                } else {
-                    appendProjectionToFrame(t, projectionList);
-                }
-
-            }
-
-        };
-    }
-}

[44/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
deleted file mode 100644
index b9b111d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ /dev/null
@@ -1,575 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class IsomorphismOperatorVisitor implements ILogicalOperatorVisitor<Boolean, ILogicalOperator> {
-
-    private final Map<LogicalVariable, LogicalVariable> variableMapping = new HashMap<LogicalVariable, LogicalVariable>();
-
-    public IsomorphismOperatorVisitor() {
-    }
-
-    @Override
-    public Boolean visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.AGGREGATE)
-            return Boolean.FALSE;
-        AggregateOperator aggOpArg = (AggregateOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(
-                getPairList(op.getVariables(), op.getExpressions()),
-                getPairList(aggOpArg.getVariables(), aggOpArg.getExpressions()));
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.RUNNINGAGGREGATE)
-            return Boolean.FALSE;
-        RunningAggregateOperator aggOpArg = (RunningAggregateOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(
-                getPairList(op.getVariables(), op.getExpressions()),
-                getPairList(aggOpArg.getVariables(), aggOpArg.getExpressions()));
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) copyAndSubstituteVar(op, arg);
-        if (aop.getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE)
-            return Boolean.FALSE;
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException {
-        ExtensionOperator aop = (ExtensionOperator) copyAndSubstituteVar(op, arg);
-        if (aop.getOperatorTag() != LogicalOperatorTag.EXTENSION_OPERATOR)
-            return Boolean.FALSE;
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        // require the same physical operator, otherwise delivers different data
-        // properties
-        if (aop.getOperatorTag() != LogicalOperatorTag.GROUP
-                || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag())
-            return Boolean.FALSE;
-
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList();
-        GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg);
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyListsArg = gbyOpArg.getGroupByList();
-
-        List<Pair<LogicalVariable, ILogicalExpression>> listLeft = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
-        List<Pair<LogicalVariable, ILogicalExpression>> listRight = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
-
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyLists)
-            listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyListsArg)
-            listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
-
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight);
-
-        if (!isomorphic)
-            return Boolean.FALSE;
-        int sizeOp = op.getNestedPlans().size();
-        int sizeArg = gbyOpArg.getNestedPlans().size();
-        if (sizeOp != sizeArg)
-            return Boolean.FALSE;
-
-        GroupByOperator argOp = (GroupByOperator) arg;
-        List<ILogicalPlan> plans = op.getNestedPlans();
-        List<ILogicalPlan> plansArg = argOp.getNestedPlans();
-        for (int i = 0; i < plans.size(); i++) {
-            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
-            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
-            if (roots.size() != rootsArg.size())
-                return Boolean.FALSE;
-            for (int j = 0; j < roots.size(); j++) {
-                ILogicalOperator topOp1 = roots.get(j).getValue();
-                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
-                isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
-                if (!isomorphic)
-                    return Boolean.FALSE;
-            }
-        }
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.LIMIT)
-            return Boolean.FALSE;
-        LimitOperator limitOpArg = (LimitOperator) copyAndSubstituteVar(op, arg);
-        if (op.getOffset() != limitOpArg.getOffset())
-            return Boolean.FALSE;
-        boolean isomorphic = op.getMaxObjects().getValue().equals(limitOpArg.getMaxObjects().getValue());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.INNERJOIN)
-            return Boolean.FALSE;
-        InnerJoinOperator joinOpArg = (InnerJoinOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN)
-            return Boolean.FALSE;
-        LeftOuterJoinOperator joinOpArg = (LeftOuterJoinOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE)
-            return Boolean.FALSE;
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.ORDER)
-            return Boolean.FALSE;
-        OrderOperator orderOpArg = (OrderOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = compareIOrderAndExpressions(op.getOrderExpressions(), orderOpArg.getOrderExpressions());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.ASSIGN)
-            return Boolean.FALSE;
-        AssignOperator assignOpArg = (AssignOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(
-                getPairList(op.getVariables(), op.getExpressions()),
-                getPairList(assignOpArg.getVariables(), assignOpArg.getExpressions()));
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.SELECT)
-            return Boolean.FALSE;
-        SelectOperator selectOpArg = (SelectOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getCondition().getValue().equals(selectOpArg.getCondition().getValue());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.PROJECT)
-            return Boolean.FALSE;
-        ProjectOperator projectOpArg = (ProjectOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), projectOpArg.getVariables());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.PARTITIONINGSPLIT)
-            return Boolean.FALSE;
-        PartitioningSplitOperator partitionOpArg = (PartitioningSplitOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = compareExpressions(op.getExpressions(), partitionOpArg.getExpressions());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.REPLICATE)
-            return Boolean.FALSE;
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.MATERIALIZE)
-            return Boolean.FALSE;
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.SCRIPT)
-            return Boolean.FALSE;
-        ScriptOperator scriptOpArg = (ScriptOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getScriptDescription().equals(scriptOpArg.getScriptDescription());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.SUBPLAN)
-            return Boolean.FALSE;
-        SubplanOperator subplanOpArg = (SubplanOperator) copyAndSubstituteVar(op, arg);
-        List<ILogicalPlan> plans = op.getNestedPlans();
-        List<ILogicalPlan> plansArg = subplanOpArg.getNestedPlans();
-        for (int i = 0; i < plans.size(); i++) {
-            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
-            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
-            if (roots.size() == rootsArg.size())
-                return Boolean.FALSE;
-            for (int j = 0; j < roots.size(); j++) {
-                ILogicalOperator topOp1 = roots.get(j).getValue();
-                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
-                boolean isomorphic = IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2);
-                if (!isomorphic)
-                    return Boolean.FALSE;
-            }
-        }
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.UNIONALL)
-            return Boolean.FALSE;
-        UnionAllOperator unionOpArg = (UnionAllOperator) copyAndSubstituteVar(op, arg);
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mapping = op.getVariableMappings();
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mappingArg = unionOpArg.getVariableMappings();
-        if (mapping.size() != mappingArg.size())
-            return Boolean.FALSE;
-        return VariableUtilities.varListEqualUnordered(mapping, mappingArg);
-    }
-
-    @Override
-    public Boolean visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST)
-            return Boolean.FALSE;
-        UnnestOperator unnestOpArg = (UnnestOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables())
-                && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable());
-        if (!isomorphic)
-            return Boolean.FALSE;
-        isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST_MAP)
-            return Boolean.FALSE;
-        UnnestMapOperator unnestOpArg = (UnnestMapOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables());
-        if (!isomorphic)
-            return Boolean.FALSE;
-        isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
-            return Boolean.FALSE;
-        DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
-        if (!argScan.getDataSource().toString().equals(op.getDataSource().toString()))
-            return Boolean.FALSE;
-        DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), scanOpArg.getVariables())
-                && op.getDataSource().toString().equals(scanOpArg.getDataSource().toString());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.DISTINCT)
-            return Boolean.FALSE;
-        DistinctOperator distinctOpArg = (DistinctOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = compareExpressions(op.getExpressions(), distinctOpArg.getExpressions());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
-            return Boolean.FALSE;
-        // require the same partition property
-        if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag()))
-            return Boolean.FALSE;
-        variableMapping.clear();
-        IsomorphismUtilities.mapVariablesTopDown(op, arg, variableMapping);
-        IPhysicalPropertiesVector properties = op.getPhysicalOperator().getDeliveredProperties();
-        IPhysicalPropertiesVector propertiesArg = aop.getPhysicalOperator().getDeliveredProperties();
-        if (properties == null && propertiesArg == null)
-            return Boolean.TRUE;
-        if (properties == null || propertiesArg == null)
-            return Boolean.FALSE;
-        IPartitioningProperty partProp = properties.getPartitioningProperty();
-        IPartitioningProperty partPropArg = propertiesArg.getPartitioningProperty();
-        if (!partProp.getPartitioningType().equals(partPropArg.getPartitioningType()))
-            return Boolean.FALSE;
-        List<LogicalVariable> columns = new ArrayList<LogicalVariable>();
-        partProp.getColumns(columns);
-        List<LogicalVariable> columnsArg = new ArrayList<LogicalVariable>();
-        partPropArg.getColumns(columnsArg);
-        if (columns.size() != columnsArg.size())
-            return Boolean.FALSE;
-        if (columns.size() == 0)
-            return Boolean.TRUE;
-        for (int i = 0; i < columnsArg.size(); i++) {
-            LogicalVariable rightVar = columnsArg.get(i);
-            LogicalVariable leftVar = variableMapping.get(rightVar);
-            if (leftVar != null)
-                columnsArg.set(i, leftVar);
-        }
-        return VariableUtilities.varListEqualUnordered(columns, columnsArg);
-    }
-
-    @Override
-    public Boolean visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.WRITE)
-            return Boolean.FALSE;
-        WriteOperator writeOpArg = (WriteOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitDistributeResultOperator(DistributeResultOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT)
-            return Boolean.FALSE;
-        DistributeResultOperator writeOpArg = (DistributeResultOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT)
-            return Boolean.FALSE;
-        WriteResultOperator writeOpArg = (WriteResultOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
-        if (!op.getDataSource().equals(writeOpArg.getDataSource()))
-            isomorphic = false;
-        if (!op.getPayloadExpression().equals(writeOpArg.getPayloadExpression()))
-            isomorphic = false;
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitInsertDeleteOperator(InsertDeleteOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE)
-            return Boolean.FALSE;
-        InsertDeleteOperator insertOpArg = (InsertDeleteOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema());
-        if (!op.getDataSource().equals(insertOpArg.getDataSource()))
-            isomorphic = false;
-        if (!op.getPayloadExpression().equals(insertOpArg.getPayloadExpression()))
-            isomorphic = false;
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.INDEX_INSERT_DELETE)
-            return Boolean.FALSE;
-        IndexInsertDeleteOperator insertOpArg = (IndexInsertDeleteOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema());
-        if (!op.getDataSourceIndex().equals(insertOpArg.getDataSourceIndex()))
-            isomorphic = false;
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitTokenizeOperator(TokenizeOperator op, ILogicalOperator arg) throws AlgebricksException {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
-        if (aop.getOperatorTag() != LogicalOperatorTag.TOKENIZE)
-            return Boolean.FALSE;
-        TokenizeOperator tokenizeOpArg = (TokenizeOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), tokenizeOpArg.getSchema());
-        if (!op.getDataSourceIndex().equals(tokenizeOpArg.getDataSourceIndex()))
-            isomorphic = false;
-        return isomorphic;
-    }
-
-    @Override
-    public Boolean visitSinkOperator(SinkOperator op, ILogicalOperator arg) throws AlgebricksException {
-        return true;
-    }
-
-    private Boolean compareExpressions(List<Mutable<ILogicalExpression>> opExprs,
-            List<Mutable<ILogicalExpression>> argExprs) {
-        if (opExprs.size() != argExprs.size())
-            return Boolean.FALSE;
-        for (int i = 0; i < opExprs.size(); i++) {
-            boolean isomorphic = opExprs.get(i).getValue().equals(argExprs.get(i).getValue());
-            if (!isomorphic)
-                return Boolean.FALSE;
-        }
-        return Boolean.TRUE;
-    }
-
-    @Override
-    public Boolean visitExternalDataLookupOperator(ExternalDataLookupOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        return Boolean.FALSE;
-    }
-
-    private Boolean compareIOrderAndExpressions(List<Pair<IOrder, Mutable<ILogicalExpression>>> opOrderExprs,
-            List<Pair<IOrder, Mutable<ILogicalExpression>>> argOrderExprs) {
-        if (opOrderExprs.size() != argOrderExprs.size())
-            return Boolean.FALSE;
-        for (int i = 0; i < opOrderExprs.size(); i++) {
-            boolean isomorphic = opOrderExprs.get(i).first.equals(argOrderExprs.get(i).first);
-            if (!isomorphic)
-                return Boolean.FALSE;
-            isomorphic = opOrderExprs.get(i).second.getValue().equals(argOrderExprs.get(i).second.getValue());
-            if (!isomorphic)
-                return Boolean.FALSE;
-        }
-        return Boolean.TRUE;
-    }
-
-    private ILogicalOperator copyAndSubstituteVar(ILogicalOperator op, ILogicalOperator argOp)
-            throws AlgebricksException {
-        ILogicalOperator newOp = OperatorManipulationUtil.deepCopy(argOp);
-        variableMapping.clear();
-        IsomorphismUtilities.mapVariablesTopDown(op, argOp, variableMapping);
-
-        List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
-        if (argOp.getInputs().size() > 0)
-            for (int i = 0; i < argOp.getInputs().size(); i++)
-                VariableUtilities.getLiveVariables(argOp.getInputs().get(i).getValue(), liveVars);
-        List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(argOp, producedVars);
-        List<LogicalVariable> producedVarsNew = new ArrayList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(op, producedVarsNew);
-
-        if (producedVars.size() != producedVarsNew.size())
-            return newOp;
-        for (Entry<LogicalVariable, LogicalVariable> map : variableMapping.entrySet()) {
-            if (liveVars.contains(map.getKey())) {
-                VariableUtilities.substituteVariables(newOp, map.getKey(), map.getValue(), null);
-            }
-        }
-        for (int i = 0; i < producedVars.size(); i++)
-            VariableUtilities.substituteVariables(newOp, producedVars.get(i), producedVarsNew.get(i), null);
-        return newOp;
-    }
-
-    public List<Pair<LogicalVariable, ILogicalExpression>> getPairList(List<LogicalVariable> vars,
-            List<Mutable<ILogicalExpression>> exprs) throws AlgebricksException {
-        List<Pair<LogicalVariable, ILogicalExpression>> list = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
-        if (vars.size() != exprs.size())
-            throw new AlgebricksException("variable list size does not equal to expression list size ");
-        for (int i = 0; i < vars.size(); i++) {
-            list.add(new Pair<LogicalVariable, ILogicalExpression>(vars.get(i), exprs.get(i).getValue()));
-        }
-        return list;
-    }
-
-    private static boolean variableEqual(LogicalVariable var, LogicalVariable varArg) {
-        if (var == null && varArg == null)
-            return true;
-        if (var.equals(varArg))
-            return true;
-        else
-            return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
deleted file mode 100644
index d79853d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public class IsomorphismUtilities {
-
-    public static void mapVariablesTopDown(ILogicalOperator op, ILogicalOperator arg,
-            Map<LogicalVariable, LogicalVariable> variableMapping) throws AlgebricksException {
-        IsomorphismVariableMappingVisitor visitor = new IsomorphismVariableMappingVisitor(variableMapping);
-        op.accept(visitor, arg);
-    }
-
-    public static boolean isOperatorIsomorphic(ILogicalOperator op, ILogicalOperator arg) throws AlgebricksException {
-        IsomorphismOperatorVisitor visitor = new IsomorphismOperatorVisitor();
-        return op.accept(visitor, arg).booleanValue();
-    }
-
-    public static boolean isOperatorIsomorphicPlanSegment(ILogicalOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> inputs1 = op.getInputs();
-        List<Mutable<ILogicalOperator>> inputs2 = arg.getInputs();
-        if (inputs1.size() != inputs2.size())
-            return Boolean.FALSE;
-        for (int i = 0; i < inputs1.size(); i++) {
-            ILogicalOperator input1 = inputs1.get(i).getValue();
-            ILogicalOperator input2 = inputs2.get(i).getValue();
-            boolean isomorphic = isOperatorIsomorphicPlanSegment(input1, input2);
-            if (!isomorphic)
-                return Boolean.FALSE;
-        }
-        return IsomorphismUtilities.isOperatorIsomorphic(op, arg);
-    }
-
-    public static boolean isOperatorIsomorphicPlan(ILogicalPlan plan, ILogicalPlan arg) throws AlgebricksException {
-        if (plan.getRoots().size() != arg.getRoots().size()) {
-            return false;
-        }
-        for (int i = 0; i < plan.getRoots().size(); i++) {
-            ILogicalOperator topOp1 = plan.getRoots().get(i).getValue();
-            ILogicalOperator topOp2 = arg.getRoots().get(i).getValue();
-            if (!IsomorphismUtilities.isOperatorIsomorphicPlanSegment(topOp1, topOp2)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
deleted file mode 100644
index 99df45c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
+++ /dev/null
@@ -1,447 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class IsomorphismVariableMappingVisitor implements ILogicalOperatorVisitor<Void, ILogicalOperator> {
-
-    private Map<LogicalVariable, LogicalVariable> variableMapping;
-
-    public IsomorphismVariableMappingVisitor(Map<LogicalVariable, LogicalVariable> variableMapping) {
-        this.variableMapping = variableMapping;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariablesForAbstractAssign(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariablesForAbstractAssign(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariablesForGroupBy(op, arg);
-        mapVariablesInNestedPlans(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        ILogicalOperator inputToCreator1 = op.getSourceOperator();
-        NestedTupleSourceOperator nts = (NestedTupleSourceOperator) arg;
-        ILogicalOperator inputToCreator2 = nts.getSourceOperator();
-        inputToCreator1.accept(this, inputToCreator2);
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariablesForAbstractAssign(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariablesInNestedPlans(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariablesForUnion(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    private void mapChildren(ILogicalOperator op, ILogicalOperator opArg) throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> inputs = op.getInputs();
-        List<Mutable<ILogicalOperator>> inputsArg = opArg.getInputs();
-        if (inputs.size() != inputsArg.size())
-            throw new AlgebricksException("children are not isomoprhic");
-        for (int i = 0; i < inputs.size(); i++) {
-            ILogicalOperator input = inputs.get(i).getValue();
-            ILogicalOperator inputArg = inputsArg.get(i).getValue();
-            input.accept(this, inputArg);
-        }
-    }
-
-    private void mapVariables(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
-        List<LogicalVariable> producedVarLeft = new ArrayList<LogicalVariable>();
-        List<LogicalVariable> producedVarRight = new ArrayList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(left, producedVarLeft);
-        VariableUtilities.getProducedVariables(right, producedVarRight);
-        mapVariables(producedVarLeft, producedVarRight);
-    }
-
-    private void mapVariables(List<LogicalVariable> variablesLeft, List<LogicalVariable> variablesRight) {
-        if (variablesLeft.size() != variablesRight.size())
-            return;
-        int size = variablesLeft.size();
-        for (int i = 0; i < size; i++) {
-            LogicalVariable left = variablesLeft.get(i);
-            LogicalVariable right = variablesRight.get(i);
-            variableMapping.put(right, left);
-        }
-    }
-
-    private void mapVariablesForAbstractAssign(ILogicalOperator left, ILogicalOperator right)
-            throws AlgebricksException {
-        AbstractAssignOperator leftOp = (AbstractAssignOperator) left;
-        AbstractAssignOperator rightOp = (AbstractAssignOperator) right;
-        List<LogicalVariable> producedVarLeft = new ArrayList<LogicalVariable>();
-        List<LogicalVariable> producedVarRight = new ArrayList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(left, producedVarLeft);
-        VariableUtilities.getProducedVariables(right, producedVarRight);
-        mapVariablesForAbstractAssign(producedVarLeft, leftOp.getExpressions(), producedVarRight,
-                rightOp.getExpressions());
-    }
-
-    private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
-        GroupByOperator leftOp = (GroupByOperator) left;
-        GroupByOperator rightOp = (GroupByOperator) right;
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList();
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList();
-        mapVarExprPairList(leftPairs, rightPairs);
-        leftPairs = leftOp.getDecorList();
-        rightPairs = rightOp.getDecorList();
-        mapVarExprPairList(leftPairs, rightPairs);
-    }
-
-    private void mapVarExprPairList(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs,
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs) {
-        if (leftPairs.size() != rightPairs.size())
-            return;
-        for (int i = 0; i < leftPairs.size(); i++) {
-            ILogicalExpression exprLeft = leftPairs.get(i).second.getValue();
-            LogicalVariable leftVar = leftPairs.get(i).first;
-            for (int j = 0; j < leftPairs.size(); j++) {
-                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(rightPairs.get(j).second).getValue();
-                if (exprLeft.equals(exprRight)) {
-                    LogicalVariable rightVar = rightPairs.get(j).first;
-                    if (rightVar != null && leftVar != null) {
-                        variableMapping.put(rightVar, leftVar);
-                    }
-                    break;
-                }
-            }
-        }
-    }
-
-    private void mapVariablesForAbstractAssign(List<LogicalVariable> variablesLeft,
-            List<Mutable<ILogicalExpression>> exprsLeft, List<LogicalVariable> variablesRight,
-            List<Mutable<ILogicalExpression>> exprsRight) {
-        if (variablesLeft.size() != variablesRight.size())
-            return;
-        int size = variablesLeft.size();
-        for (int i = 0; i < size; i++) {
-            ILogicalExpression exprLeft = exprsLeft.get(i).getValue();
-            LogicalVariable left = variablesLeft.get(i);
-            for (int j = 0; j < size; j++) {
-                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(exprsRight.get(j)).getValue();
-                if (exprLeft.equals(exprRight)) {
-                    LogicalVariable right = variablesRight.get(j);
-                    variableMapping.put(right, left);
-                    break;
-                }
-            }
-        }
-    }
-
-    private void mapVariablesInNestedPlans(ILogicalOperator opOrigin, ILogicalOperator arg) throws AlgebricksException {
-        AbstractOperatorWithNestedPlans op = (AbstractOperatorWithNestedPlans) opOrigin;
-        AbstractOperatorWithNestedPlans argOp = (AbstractOperatorWithNestedPlans) arg;
-        List<ILogicalPlan> plans = op.getNestedPlans();
-        List<ILogicalPlan> plansArg = argOp.getNestedPlans();
-        if (plans.size() != plansArg.size())
-            return;
-        for (int i = 0; i < plans.size(); i++) {
-            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
-            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
-            if (roots.size() != rootsArg.size())
-                return;
-            for (int j = 0; j < roots.size(); j++) {
-                ILogicalOperator topOp1 = roots.get(j).getValue();
-                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
-                topOp1.accept(this, topOp2);
-            }
-        }
-    }
-
-    private void mapVariablesStandard(ILogicalOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapChildren(op, arg);
-        mapVariables(op, arg);
-    }
-
-    private Mutable<ILogicalExpression> copyExpressionAndSubtituteVars(Mutable<ILogicalExpression> expr) {
-        ILogicalExpression copy = ((AbstractLogicalExpression) expr.getValue()).cloneExpression();
-        for (Entry<LogicalVariable, LogicalVariable> entry : variableMapping.entrySet())
-            copy.substituteVar(entry.getKey(), entry.getValue());
-        return new MutableObject<ILogicalExpression>(copy);
-    }
-
-    private void mapVariablesForUnion(ILogicalOperator op, ILogicalOperator arg) {
-        UnionAllOperator union = (UnionAllOperator) op;
-        UnionAllOperator unionArg = (UnionAllOperator) arg;
-        mapVarTripleList(union.getVariableMappings(), unionArg.getVariableMappings());
-    }
-
-    private void mapVarTripleList(List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> leftTriples,
-            List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> rightTriples) {
-        if (leftTriples.size() != rightTriples.size())
-            return;
-        for (int i = 0; i < leftTriples.size(); i++) {
-            LogicalVariable leftFirstVar = leftTriples.get(i).first;
-            LogicalVariable leftSecondVar = leftTriples.get(i).second;
-            LogicalVariable leftThirdVar = leftTriples.get(i).third;
-            for (int j = 0; j < rightTriples.size(); j++) {
-                LogicalVariable rightFirstVar = rightTriples.get(j).first;
-                LogicalVariable rightSecondVar = rightTriples.get(j).second;
-                LogicalVariable rightThirdVar = rightTriples.get(j).third;
-                if (varEquivalent(leftFirstVar, rightFirstVar) && varEquivalent(leftSecondVar, rightSecondVar)) {
-                    variableMapping.put(rightThirdVar, leftThirdVar);
-                    break;
-                }
-            }
-        }
-    }
-
-    private boolean varEquivalent(LogicalVariable left, LogicalVariable right) {
-        if (variableMapping.get(right) == null)
-            return false;
-        if (variableMapping.get(right).equals(left))
-            return true;
-        else
-            return false;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException {
-        mapVariablesStandard(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, ILogicalOperator arg)
-            throws AlgebricksException {
-        return null;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
deleted file mode 100644
index 19fcf65..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LogicalPropertiesVectorImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-
-public class LogicalPropertiesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
-
-    public static void computeLogicalPropertiesDFS(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        LogicalPropertiesVisitor visitor = new LogicalPropertiesVisitor();
-        computeLogicalPropertiesRec(op, visitor, context);
-    }
-
-    private static void computeLogicalPropertiesRec(ILogicalOperator op, LogicalPropertiesVisitor visitor,
-            IOptimizationContext context) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> ref : op.getInputs()) {
-            computeLogicalPropertiesRec(ref.getValue(), visitor, context);
-        }
-        op.accept(visitor, context);
-        if (AlgebricksConfig.DEBUG) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.finest("Logical properties visitor for " + op + ": "
-                    + context.getLogicalPropertiesVector(op) + "\n");
-        }
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, IOptimizationContext context) throws AlgebricksException {
-        visitAssignment(op, context);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, IOptimizationContext arg) throws AlgebricksException {
-        propagateCardinalityAndFrameNumber(op, arg);
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, IOptimizationContext arg) throws AlgebricksException {
-        Object annot1 = op.getAnnotations().get(OperatorAnnotations.CARDINALITY);
-        if (annot1 == null) {
-            return null;
-        }
-        Integer m = (Integer) annot1;
-        LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl();
-        v.setNumberOfTuples(m);
-        Object annot2 = op.getAnnotations().get(OperatorAnnotations.MAX_NUMBER_FRAMES);
-        if (annot2 != null) {
-            Integer f = (Integer) annot2;
-            v.setMaxOutputFrames(f);
-        }
-        arg.putLogicalPropertiesVector(op, v);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, IOptimizationContext context) throws AlgebricksException {
-        propagateCardinalityAndFrameNumber(op, context);
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        visitAssignment(op, context);
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    private LogicalPropertiesVectorImpl propagateCardinality(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op0 = op.getInputs().get(0).getValue();
-        ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
-        if (v0 == null) {
-            return null;
-        }
-        LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl();
-        v.setNumberOfTuples(v0.getNumberOfTuples());
-        context.putLogicalPropertiesVector(op, v);
-        return v;
-    }
-
-    private void visitAssignment(AbstractAssignOperator op, IOptimizationContext context) throws AlgebricksException {
-        LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
-        if (v != null && v.getNumberOfTuples() != null) {
-            IVariableEvalSizeEnvironment varSizeEnv = context.getVariableEvalSizeEnvironment();
-            IExpressionEvalSizeComputer evalSize = context.getExpressionEvalSizeComputer();
-            if (evalSize != null) {
-                ILogicalOperator op0 = op.getInputs().get(0).getValue();
-                ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
-                if (v0 != null) {
-                    long frames0 = v0.getMaxOutputFrames();
-                    long overhead = 0; // added per tuple
-                    for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-                        int sz = evalSize.getEvalSize(exprRef.getValue(), varSizeEnv);
-                        if (sz == -1) {
-                            return;
-                        }
-                        overhead += sz;
-                    }
-                    int frameSize = context.getPhysicalOptimizationConfig().getFrameSize();
-                    if (frameSize > 0) {
-                        long sz = frames0 * frameSize + overhead * v.getNumberOfTuples();
-                        int frames1 = (int) (sz / frameSize);
-                        if (sz % frameSize > 0) {
-                            frames1++;
-                        }
-                        v.setMaxOutputFrames(frames1);
-                    }
-                }
-            }
-        }
-    }
-
-    public void propagateCardinalityAndFrameNumber(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
-        // propagate also max number of frames (conservatively)
-        ILogicalOperator op0 = op.getInputs().get(0).getValue();
-        ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
-        if (v0 != null) {
-            v.setMaxOutputFrames(v0.getMaxOutputFrames());
-        }
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, IOptimizationContext arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, IOptimizationContext arg)
-            throws AlgebricksException {
-        return null;
-    }
-
-}

[05/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java
new file mode 100644
index 0000000..1eef9f8
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanIntoGroupByRule.java
@@ -0,0 +1,217 @@
+/*
+
+ * Copyright 2009-2013 by The Regents of the University of California
+
+ * Licensed under the Apache License, Version 2.0 (the "License");
+
+ * you may not use this file except in compliance with the License.
+
+ * you may obtain a copy of the License from
+
+ * 
+
+ *     http://www.apache.org/licenses/LICENSE-2.0
+
+ * 
+
+ * Unless required by applicable law or agreed to in writing, software
+
+ * distributed under the License is distributed on an "AS IS" BASIS,
+
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+ * See the License for the specific language governing permissions and
+
+ * limitations under the License.
+
+ */
+
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule pushes an array of subplans on top of a group-by into the
+ * nested plan of the group-by.
+ * 
+ * @author yingyib
+ */
+
+public class PushSubplanIntoGroupByRule implements IAlgebraicRewriteRule {
+    /** Stores used variables above the current operator. */
+    private final Set<LogicalVariable> usedVarsSoFar = new HashSet<LogicalVariable>();
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        ILogicalOperator parentOperator = opRef.getValue();
+        if (context.checkIfInDontApplySet(this, parentOperator)) {
+            return false;
+        }
+        context.addToDontApplySet(this, parentOperator);
+        VariableUtilities.getUsedVariables(parentOperator, usedVarsSoFar);
+        if (parentOperator.getInputs().size() <= 0) {
+            return false;
+        }
+        boolean changed = false;
+        GroupByOperator gby = null;
+        for (Mutable<ILogicalOperator> ref : parentOperator.getInputs()) {
+            AbstractLogicalOperator op = (AbstractLogicalOperator) ref.getValue();
+            /** Only processes subplan operator. */
+            List<SubplanOperator> subplans = new ArrayList<SubplanOperator>();
+            if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+                while (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+                    SubplanOperator currentSubplan = (SubplanOperator) op;
+                    subplans.add(currentSubplan);
+                    op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+                }
+                /** Only processes the case a group-by operator is the input of the subplan operators. */
+                if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
+                    gby = (GroupByOperator) op;
+                    List<ILogicalPlan> newGbyNestedPlans = new ArrayList<ILogicalPlan>();
+                    for (SubplanOperator subplan : subplans) {
+                        List<ILogicalPlan> subplanNestedPlans = subplan.getNestedPlans();
+                        List<ILogicalPlan> gbyNestedPlans = gby.getNestedPlans();
+                        List<ILogicalPlan> subplanNestedPlansToRemove = new ArrayList<ILogicalPlan>();
+                        for (ILogicalPlan subplanNestedPlan : subplanNestedPlans) {
+                            List<Mutable<ILogicalOperator>> rootOpRefs = subplanNestedPlan.getRoots();
+                            List<Mutable<ILogicalOperator>> rootOpRefsToRemove = new ArrayList<Mutable<ILogicalOperator>>();
+                            for (Mutable<ILogicalOperator> rootOpRef : rootOpRefs) {
+                                /** Gets free variables in the root operator of a nested plan and its descent. */
+                                Set<LogicalVariable> freeVars = new ListSet<LogicalVariable>();
+                                VariableUtilities.getUsedVariablesInDescendantsAndSelf(rootOpRef.getValue(), freeVars);
+                                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
+                                VariableUtilities.getProducedVariablesInDescendantsAndSelf(rootOpRef.getValue(),
+                                        producedVars);
+                                freeVars.removeAll(producedVars);
+                                /** * Checks whether the above freeVars are all contained in live variables * of one nested plan inside the group-by operator. * If yes, then the subplan can be pushed into the nested plan of the group-by. */
+                                for (ILogicalPlan gbyNestedPlanOriginal : gbyNestedPlans) {
+                                    // add a subplan in the original gby
+                                    if (!newGbyNestedPlans.contains(gbyNestedPlanOriginal)) {
+                                        newGbyNestedPlans.add(gbyNestedPlanOriginal);
+                                    }
+
+                                    // add a pushed subplan
+                                    ILogicalPlan gbyNestedPlan = OperatorManipulationUtil.deepCopy(
+                                            gbyNestedPlanOriginal, context);
+                                    List<Mutable<ILogicalOperator>> gbyRootOpRefs = gbyNestedPlan.getRoots();
+                                    for (int rootIndex = 0; rootIndex < gbyRootOpRefs.size(); rootIndex++) {
+                                        //set the nts for a original subplan
+                                        Mutable<ILogicalOperator> originalGbyRootOpRef = gbyNestedPlanOriginal
+                                                .getRoots().get(rootIndex);
+                                        Mutable<ILogicalOperator> originalGbyNtsRef = downToNts(originalGbyRootOpRef);
+                                        NestedTupleSourceOperator originalNts = (NestedTupleSourceOperator) originalGbyNtsRef
+                                                .getValue();
+                                        originalNts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
+
+                                        //push a new subplan if possible
+                                        Mutable<ILogicalOperator> gbyRootOpRef = gbyRootOpRefs.get(rootIndex);
+                                        Set<LogicalVariable> liveVars = new ListSet<LogicalVariable>();
+                                        VariableUtilities.getLiveVariables(gbyRootOpRef.getValue(), liveVars);
+                                        if (liveVars.containsAll(freeVars)) {
+                                            /** Does the actual push. */
+                                            Mutable<ILogicalOperator> ntsRef = downToNts(rootOpRef);
+                                            ntsRef.setValue(gbyRootOpRef.getValue());
+                                            // Removes unused vars.
+                                            AggregateOperator aggOp = (AggregateOperator) gbyRootOpRef.getValue();
+                                            for (int varIndex = aggOp.getVariables().size() - 1; varIndex >= 0; varIndex--) {
+                                                if (!freeVars.contains(aggOp.getVariables().get(varIndex))) {
+                                                    aggOp.getVariables().remove(varIndex);
+                                                    aggOp.getExpressions().remove(varIndex);
+                                                }
+                                            }
+
+                                            gbyRootOpRef.setValue(rootOpRef.getValue());
+                                            rootOpRefsToRemove.add(rootOpRef);
+
+                                            // Sets the nts for a new pushed plan.
+                                            Mutable<ILogicalOperator> oldGbyNtsRef = downToNts(gbyRootOpRef);
+                                            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) oldGbyNtsRef
+                                                    .getValue();
+                                            nts.setDataSourceReference(new MutableObject<ILogicalOperator>(gby));
+
+                                            newGbyNestedPlans.add(gbyNestedPlan);
+                                            changed = true;
+                                            continue;
+                                        }
+                                    }
+                                }
+                            }
+                            rootOpRefs.removeAll(rootOpRefsToRemove);
+                            if (rootOpRefs.size() == 0) {
+                                subplanNestedPlansToRemove.add(subplanNestedPlan);
+                            }
+                        }
+                        subplanNestedPlans.removeAll(subplanNestedPlansToRemove);
+                    }
+                    if (changed) {
+                        ref.setValue(gby);
+                        gby.getNestedPlans().clear();
+                        gby.getNestedPlans().addAll(newGbyNestedPlans);
+                    }
+                }
+            }
+        }
+        if (changed) {
+            cleanup(gby);
+        }
+        return changed;
+    }
+
+    /**
+     * Removes unused aggregation variables (and expressions)
+     * 
+     * @param gby
+     * @throws AlgebricksException
+     */
+    private void cleanup(GroupByOperator gby) throws AlgebricksException {
+        for (ILogicalPlan nestedPlan : gby.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
+                AggregateOperator aggOp = (AggregateOperator) rootRef.getValue();
+                for (int varIndex = aggOp.getVariables().size() - 1; varIndex >= 0; varIndex--) {
+                    if (!usedVarsSoFar.contains(aggOp.getVariables().get(varIndex))) {
+                        aggOp.getVariables().remove(varIndex);
+                        aggOp.getExpressions().remove(varIndex);
+                    }
+                }
+            }
+
+        }
+    }
+
+    private Mutable<ILogicalOperator> downToNts(Mutable<ILogicalOperator> opRef) {
+        Mutable<ILogicalOperator> currentOpRef = opRef;
+        while (currentOpRef.getValue().getInputs().size() > 0) {
+            currentOpRef = currentOpRef.getValue().getInputs().get(0);
+        }
+        return currentOpRef;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
new file mode 100644
index 0000000..9c15f77
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushSubplanWithAggregateDownThroughProductRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op1;
+        if (subplan.getNestedPlans().size() != 1) {
+            return false;
+        }
+        ILogicalPlan p = subplan.getNestedPlans().get(0);
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> r = p.getRoots().get(0);
+        if (((AbstractLogicalOperator) r.getValue()).getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+
+        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
+
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+        ILogicalOperator b0 = b0Ref.getValue();
+        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(b0, b0Scm);
+        if (b0Scm.containsAll(free)) {
+            // push subplan on left branch
+            op2Ref.setValue(b0);
+            b0Ref.setValue(op1);
+            opRef.setValue(op2);
+            return true;
+        } else {
+            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+            ILogicalOperator b1 = b1Ref.getValue();
+            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(b1, b1Scm);
+            if (b1Scm.containsAll(free)) {
+                // push subplan on right branch
+                op2Ref.setValue(b1);
+                b1Ref.setValue(op1);
+                opRef.setValue(op2);
+                return true;
+            } else {
+                return false;
+            }
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java
new file mode 100644
index 0000000..c70fca9
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughProductRule.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushUnnestDownThroughProductRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return false;
+        }
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+        if (join.getCondition().getValue() != ConstantExpression.TRUE) {
+            return false;
+        }
+
+        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op1, used);
+
+        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+        ILogicalOperator b0 = b0Ref.getValue();
+        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(b0, b0Scm);
+        if (b0Scm.containsAll(used)) {
+            // push unnest on left branch
+            op2Ref.setValue(b0);
+            b0Ref.setValue(op1);
+            opRef.setValue(op2);
+            return true;
+        } else {
+            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+            ILogicalOperator b1 = b1Ref.getValue();
+            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(b1, b1Scm);
+            if (b1Scm.containsAll(used)) {
+                // push unnest on right branch
+                op2Ref.setValue(b1);
+                b1Ref.setValue(op1);
+                opRef.setValue(op2);
+                return true;
+            } else {
+                return false;
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java
new file mode 100644
index 0000000..b69bfb3
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushUnnestDownThroughUnionRule.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * @author kereno, ecarm002, ildar.absalyamov
+ *         Pushes down unnest through both branches of the union operator
+ *         Before rule:
+ *         ============
+ *         unnest
+ *         union (left_branch, right_branch, res)
+ *         left_branch
+ *         right_branch
+ *         After rule:
+ *         ============
+ *         union (left_branch, right_branch, res)
+ *         unnest
+ *         left_branch
+ *         unnest
+ *         right_branch
+ */
+public class PushUnnestDownThroughUnionRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+
+        AbstractLogicalOperator unnest = (AbstractLogicalOperator) opRef.getValue();
+        if (unnest.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return false;
+        }
+        UnnestOperator unnestOpRef = (UnnestOperator) opRef.getValue();
+        Mutable<ILogicalOperator> unionOp = unnest.getInputs().get(0);
+
+        AbstractLogicalOperator unionAbstractOp = (AbstractLogicalOperator) unionOp.getValue();
+        if (unionAbstractOp.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
+            return false;
+        }
+
+        LogicalVariable unnestVar1 = context.newVar();
+        UnnestOperator unnest1 = new UnnestOperator(unnestVar1, new MutableObject<ILogicalExpression>(unnestOpRef
+                .getExpressionRef().getValue().cloneExpression()));
+        LogicalVariable unnestVar2 = context.newVar();
+        UnnestOperator unnest2 = new UnnestOperator(unnestVar2, new MutableObject<ILogicalExpression>(unnestOpRef
+                .getExpressionRef().getValue().cloneExpression()));
+
+        //Getting the two topmost branched and adding them as an input to the unnests:
+        Mutable<ILogicalOperator> branch1 = unionAbstractOp.getInputs().get(0);
+        ILogicalOperator agg1 = branch1.getValue();
+        List<LogicalVariable> agg1_var = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(agg1, agg1_var);
+        Mutable<ILogicalOperator> branch2 = unionAbstractOp.getInputs().get(1);
+        ILogicalOperator agg2 = branch2.getValue();
+        List<LogicalVariable> agg2_var = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(agg2, agg2_var);
+
+        //Modifying the unnest so it has the right variable
+        List<LogicalVariable> var_unnest_1 = new ArrayList<LogicalVariable>();
+        unnest1.getExpressionRef().getValue().getUsedVariables(var_unnest_1);
+        unnest1.getExpressionRef().getValue().substituteVar(var_unnest_1.get(0), agg1_var.get(0));
+
+        List<LogicalVariable> var_unnest2 = new ArrayList<LogicalVariable>();
+        unnest2.getExpressionRef().getValue().getUsedVariables(var_unnest2);
+        unnest2.getExpressionRef().getValue().substituteVar(var_unnest2.get(0), agg2_var.get(0));
+
+        unnest1.getInputs().add(branch1);
+        unnest2.getInputs().add(branch2);
+        context.computeAndSetTypeEnvironmentForOperator(unnest1);
+        context.computeAndSetTypeEnvironmentForOperator(unnest2);
+
+        //creating a new union operator with the updated logical variables
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>(
+                1);
+        Triple<LogicalVariable, LogicalVariable, LogicalVariable> union_triple_vars = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
+                unnestVar1, unnestVar2, unnestOpRef.getVariables().get(0));
+        varMap.add(union_triple_vars);
+        UnionAllOperator unionOpFinal = new UnionAllOperator(varMap);
+
+        unionOpFinal.getInputs().add(new MutableObject<ILogicalOperator>(unnest1));
+        unionOpFinal.getInputs().add(new MutableObject<ILogicalOperator>(unnest2));
+
+        context.computeAndSetTypeEnvironmentForOperator(unionOpFinal);
+
+        opRef.setValue(unionOpFinal);
+        return true;
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
new file mode 100644
index 0000000..aac3a18
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ReinferAllTypesRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+            return false;
+        }
+        typeOpRec(opRef, context);
+        return true;
+    }
+
+    private void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
+            typeOpRec(r, context);
+        }
+    }
+
+    private void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            typeOpRec(i, context);
+        }
+        if (op.hasNestedPlans()) {
+            for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+                typePlan(p, context);
+            }
+        }
+        context.computeAndSetTypeEnvironmentForOperator(op);
+        context.addToDontApplySet(this, op);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java
new file mode 100644
index 0000000..1862834
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Removes duplicate variables from a group-by operator's decor list.
+ */
+public class RemoveRedundantGroupByDecorVars implements IAlgebraicRewriteRule {
+
+    private final Set<LogicalVariable> vars = new HashSet<LogicalVariable>();
+    
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        if (context.checkIfInDontApplySet(this, op)) {
+            return false;
+        }
+        vars.clear();
+        
+        boolean modified = false;
+        GroupByOperator groupOp = (GroupByOperator) op;
+        Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
+        while (iter.hasNext()) {
+            Pair<LogicalVariable, Mutable<ILogicalExpression>> decor = iter.next();
+            if (decor.first != null || decor.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                continue;
+            }
+            VariableReferenceExpression varRefExpr = (VariableReferenceExpression) decor.second.getValue();
+            LogicalVariable var = varRefExpr.getVariableReference();
+            if (vars.contains(var)) {
+                iter.remove();
+                modified = true;
+            } else {
+                vars.add(var);
+            }
+        }
+        if (modified) {
+            context.addToDontApplySet(this, op);
+        }
+        return modified;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
new file mode 100644
index 0000000..10091b8
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/*
+ *  project [var-list1]
+ *   project [var-list2]
+ *     P
+ * 
+ *  if var-list1.equals(var-list2) becomes
+ * 
+ *  project [var-list1]
+ *    P
+ *  
+ */
+
+public class RemoveRedundantProjectionRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+            Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+            if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+                return false;
+            }
+            ProjectOperator pi2 = (ProjectOperator) op2;
+            opRef2.setValue(pi2.getInputs().get(0).getValue());
+        } else {
+            if (op1.getInputs().size() <= 0)
+                return false;
+            Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+            if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+                return false;
+            }
+            if (op2.getInputs().size() <= 0)
+                return false;
+            Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
+            AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
+
+            List<LogicalVariable> liveVars2 = new ArrayList<LogicalVariable>();
+            List<LogicalVariable> liveVars3 = new ArrayList<LogicalVariable>();
+
+            VariableUtilities.getLiveVariables(op2, liveVars2);
+            VariableUtilities.getLiveVariables(op3, liveVars3);
+
+            if (!VariableUtilities.varListEqualUnordered(liveVars2, liveVars3))
+                return false;
+            opRef2.setValue(op3);
+        }
+
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
new file mode 100644
index 0000000..8b1aeb4
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Replaces redundant variable references with their bottom-most equivalent representative.
+ * Does a DFS sweep over the plan keeping track of variable equivalence classes.
+ * For example, this rule would perform the following rewrite.
+ * Before Plan:
+ * select (function-call: func, Args:[%0->$$11])
+ * project [$11]
+ * assign [$$11] <- [$$10]
+ * assign [$$10] <- [$$9]
+ * assign [$$9] <- ...
+ * ...
+ * After Plan:
+ * select (function-call: func, Args:[%0->$$9])
+ * project [$9]
+ * assign [$$11] <- [$$9]
+ * assign [$$10] <- [$$9]
+ * assign [$$9] <- ...
+ * ...
+ */
+public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
+
+    private final VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor();
+    private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap = new HashMap<LogicalVariable, List<LogicalVariable>>();
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+            return false;
+        }
+        boolean modified = removeRedundantVariables(opRef, context);
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
+        }
+        return modified;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    private void updateEquivalenceClassMap(LogicalVariable lhs, LogicalVariable rhs) {
+        List<LogicalVariable> equivalentVars = equivalentVarsMap.get(rhs);
+        if (equivalentVars == null) {
+            equivalentVars = new ArrayList<LogicalVariable>();
+            // The first element in the list is the bottom-most representative which will replace all equivalent vars.
+            equivalentVars.add(rhs);
+            equivalentVars.add(lhs);
+            equivalentVarsMap.put(rhs, equivalentVars);
+        }
+        equivalentVarsMap.put(lhs, equivalentVars);
+        equivalentVars.get(0);
+    }
+
+    private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        boolean modified = false;
+
+        // Update equivalence class map.
+        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+            AssignOperator assignOp = (AssignOperator) op;
+            int numVars = assignOp.getVariables().size();
+            for (int i = 0; i < numVars; i++) {
+                ILogicalExpression expr = assignOp.getExpressions().get(i).getValue();
+                if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                    continue;
+                }
+                VariableReferenceExpression rhsVarRefExpr = (VariableReferenceExpression) expr;
+                // Update equivalence class map.
+                LogicalVariable lhs = assignOp.getVariables().get(i);
+                LogicalVariable rhs = rhsVarRefExpr.getVariableReference();
+                updateEquivalenceClassMap(lhs, rhs);
+            }
+        }
+
+        // Replace variable references with their first representative. 
+        if (op.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+            // The project operator does not use expressions, so we need to replace it's variables manually.
+            if (replaceProjectVars((ProjectOperator) op)) {
+                modified = true;
+            }
+        } else if (op.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
+            // Replace redundant variables manually in the UnionAll operator.
+            if (replaceUnionAllVars((UnionAllOperator) op)) {
+                modified = true;
+            }
+        } else {
+            if (op.acceptExpressionTransform(substVisitor)) {
+                modified = true;
+            }
+        }
+
+        // Perform variable replacement in nested plans. 
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
+                    if (removeRedundantVariables(rootRef, context)) {
+                        modified = true;
+                    }
+                }
+            }
+        }
+
+        // Deal with re-mapping of variables in group by.
+        if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            if (handleGroupByVarRemapping((GroupByOperator) op)) {
+                modified = true;
+            }
+        }
+
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(op);
+            context.addToDontApplySet(this, op);
+        }
+        return modified;
+    }
+
+    private boolean handleGroupByVarRemapping(GroupByOperator groupOp) {
+        boolean modified = false;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gp : groupOp.getGroupByList()) {
+            if (gp.first == null || gp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                continue;
+            }
+            LogicalVariable groupByVar = ((VariableReferenceExpression) gp.second.getValue()).getVariableReference();
+            Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
+            while (iter.hasNext()) {
+                Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
+                if (dp.first != null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                    continue;
+                }
+                LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference();
+                if (dv == groupByVar) {
+                    // The decor variable is redundant, since it is propagated as a grouping variable.
+                    List<LogicalVariable> equivalentVars = equivalentVarsMap.get(groupByVar);
+                    if (equivalentVars != null) {
+                        // Change representative of this equivalence class.
+                        equivalentVars.set(0, gp.first);
+                        equivalentVarsMap.put(gp.first, equivalentVars);
+                    } else {
+                        updateEquivalenceClassMap(gp.first, groupByVar);
+                    }
+                    iter.remove();
+                    modified = true;
+                    break;
+                }
+            }
+        }
+        // find the redundant variables within the decor list
+        Map<LogicalVariable, LogicalVariable> variableToFirstDecorMap = new HashMap<LogicalVariable, LogicalVariable>();
+        Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
+        while (iter.hasNext()) {
+            Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
+            if (dp.first == null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                continue;
+            }
+            LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference();
+            LogicalVariable firstDecor = variableToFirstDecorMap.get(dv);
+            if (firstDecor == null) {
+                variableToFirstDecorMap.put(dv, dp.first);
+            } else {
+                // The decor variable dp.first is redundant since firstDecor is exactly the same.
+                updateEquivalenceClassMap(dp.first, firstDecor);
+                iter.remove();
+                modified = true;
+            }
+        }
+        return modified;
+    }
+
+    /**
+     * Replace the projects's variables with their corresponding representative
+     * from the equivalence class map (if any).
+     * We cannot use the VariableSubstitutionVisitor here because the project ops
+     * maintain their variables as a list and not as expressions.
+     */
+    private boolean replaceProjectVars(ProjectOperator op) throws AlgebricksException {
+        List<LogicalVariable> vars = op.getVariables();
+        int size = vars.size();
+        boolean modified = false;
+        for (int i = 0; i < size; i++) {
+            LogicalVariable var = vars.get(i);
+            List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
+            if (equivalentVars == null) {
+                continue;
+            }
+            // Replace with equivalence class representative.
+            LogicalVariable representative = equivalentVars.get(0);
+            if (representative != var) {
+                vars.set(i, equivalentVars.get(0));
+                modified = true;
+            }
+        }
+        return modified;
+    }
+
+    private boolean replaceUnionAllVars(UnionAllOperator op) throws AlgebricksException {
+        boolean modified = false;
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : op.getVariableMappings()) {
+            List<LogicalVariable> firstEquivalentVars = equivalentVarsMap.get(varMapping.first);
+            List<LogicalVariable> secondEquivalentVars = equivalentVarsMap.get(varMapping.second);
+            // Replace variables with their representative.
+            if (firstEquivalentVars != null) {
+                varMapping.first = firstEquivalentVars.get(0);
+                modified = true;
+            }
+            if (secondEquivalentVars != null) {
+                varMapping.second = secondEquivalentVars.get(0);
+                modified = true;
+            }
+        }
+        return modified;
+    }
+
+    private class VariableSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
+        @Override
+        public boolean transform(Mutable<ILogicalExpression> exprRef) {
+            ILogicalExpression e = exprRef.getValue();
+            switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+                case VARIABLE: {
+                    // Replace variable references with their equivalent representative in the equivalence class map.
+                    VariableReferenceExpression varRefExpr = (VariableReferenceExpression) e;
+                    LogicalVariable var = varRefExpr.getVariableReference();
+                    List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
+                    if (equivalentVars == null) {
+                        return false;
+                    }
+                    LogicalVariable representative = equivalentVars.get(0);
+                    if (representative != var) {
+                        varRefExpr.setVariable(representative);
+                        return true;
+                    }
+                    return false;
+                }
+                case FUNCTION_CALL: {
+                    AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) e;
+                    boolean modified = false;
+                    for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
+                        if (transform(arg)) {
+                            modified = true;
+                        }
+                    }
+                    return modified;
+                }
+                default: {
+                    return false;
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
new file mode 100644
index 0000000..de4fbfb
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -0,0 +1,240 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Removes unused variables from Assign, Unnest, Aggregate, and UnionAll operators.
+ */
+public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+            return false;
+        }
+        Set<LogicalVariable> toRemove = new HashSet<LogicalVariable>();
+        collectUnusedAssignedVars((AbstractLogicalOperator) opRef.getValue(), toRemove, true, context);
+        boolean smthToRemove = !toRemove.isEmpty();
+        if (smthToRemove) {
+            removeUnusedAssigns(opRef, toRemove, context);
+        }
+        return !toRemove.isEmpty();
+    }
+
+    private void removeUnusedAssigns(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> toRemove,
+            IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        while (removeFromAssigns(op, toRemove, context) == 0) {
+            if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+                break;
+            }
+            op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+            opRef.setValue(op);
+        }
+        Iterator<Mutable<ILogicalOperator>> childIter = op.getInputs().iterator();
+        while (childIter.hasNext()) {
+            Mutable<ILogicalOperator> cRef = childIter.next();
+            removeUnusedAssigns(cRef, toRemove, context);
+        }
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans opWithNest = (AbstractOperatorWithNestedPlans) op;
+            Iterator<ILogicalPlan> planIter = opWithNest.getNestedPlans().iterator();
+            while (planIter.hasNext()) {
+                ILogicalPlan p = planIter.next();
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    removeUnusedAssigns(r, toRemove, context);
+                }
+            }
+
+            // Removes redundant nested plans that produces nothing
+            for (int i = opWithNest.getNestedPlans().size() - 1; i >= 0; i--) {
+                ILogicalPlan nestedPlan = opWithNest.getNestedPlans().get(i);
+                List<Mutable<ILogicalOperator>> rootsToBeRemoved = new ArrayList<Mutable<ILogicalOperator>>();
+                for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
+                    ILogicalOperator topOp = r.getValue();
+                    Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
+                    VariableUtilities.getProducedVariablesInDescendantsAndSelf(topOp, producedVars);
+                    if (producedVars.size() == 0) {
+                        rootsToBeRemoved.add(r);
+                    }
+                }
+                // Makes sure the operator should have at least ONE nested plan even it is empty 
+                // (because a lot of places uses this assumption,  TODO(yingyib): clean them up).
+                if (nestedPlan.getRoots().size() == rootsToBeRemoved.size() && opWithNest.getNestedPlans().size() > 1) {
+                    nestedPlan.getRoots().removeAll(rootsToBeRemoved);
+                    opWithNest.getNestedPlans().remove(nestedPlan);
+                }
+            }
+        }
+    }
+
+    private int removeFromAssigns(AbstractLogicalOperator op, Set<LogicalVariable> toRemove,
+            IOptimizationContext context) throws AlgebricksException {
+        switch (op.getOperatorTag()) {
+            case ASSIGN: {
+                AssignOperator assign = (AssignOperator) op;
+                if (removeUnusedVarsAndExprs(toRemove, assign.getVariables(), assign.getExpressions())) {
+                    context.computeAndSetTypeEnvironmentForOperator(assign);
+                }
+                return assign.getVariables().size();
+            }
+            case AGGREGATE: {
+                AggregateOperator agg = (AggregateOperator) op;
+                if (removeUnusedVarsAndExprs(toRemove, agg.getVariables(), agg.getExpressions())) {
+                    context.computeAndSetTypeEnvironmentForOperator(agg);
+                }
+                return agg.getVariables().size();
+            }
+            case UNNEST: {
+                UnnestOperator uOp = (UnnestOperator) op;
+                LogicalVariable pVar = uOp.getPositionalVariable();
+                if (pVar != null && toRemove.contains(pVar)) {
+                    uOp.setPositionalVariable(null);
+                }
+                break;
+            }
+            case UNIONALL: {
+                UnionAllOperator unionOp = (UnionAllOperator) op;
+                if (removeUnusedVarsFromUnionAll(unionOp, toRemove)) {
+                    context.computeAndSetTypeEnvironmentForOperator(unionOp);
+                }
+                return unionOp.getVariableMappings().size();
+            }
+        }
+        return -1;
+    }
+
+    private boolean removeUnusedVarsFromUnionAll(UnionAllOperator unionOp, Set<LogicalVariable> toRemove) {
+        Iterator<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> iter = unionOp.getVariableMappings()
+                .iterator();
+        boolean modified = false;
+        Set<LogicalVariable> removeFromRemoveSet = new HashSet<LogicalVariable>();
+        while (iter.hasNext()) {
+            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping = iter.next();
+            if (toRemove.contains(varMapping.third)) {
+                iter.remove();
+                modified = true;
+            }
+            // In any case, make sure we do not removing these variables.
+            removeFromRemoveSet.add(varMapping.first);
+            removeFromRemoveSet.add(varMapping.second);
+        }
+        toRemove.removeAll(removeFromRemoveSet);
+        return modified;
+    }
+
+    private boolean removeUnusedVarsAndExprs(Set<LogicalVariable> toRemove, List<LogicalVariable> varList,
+            List<Mutable<ILogicalExpression>> exprList) {
+        boolean changed = false;
+        Iterator<LogicalVariable> varIter = varList.iterator();
+        Iterator<Mutable<ILogicalExpression>> exprIter = exprList.iterator();
+        while (varIter.hasNext()) {
+            LogicalVariable v = varIter.next();
+            exprIter.next();
+            if (toRemove.contains(v)) {
+                varIter.remove();
+                exprIter.remove();
+                changed = true;
+            }
+        }
+        return changed;
+    }
+
+    private void collectUnusedAssignedVars(AbstractLogicalOperator op, Set<LogicalVariable> toRemove, boolean first,
+            IOptimizationContext context) throws AlgebricksException {
+        if (!first) {
+            context.addToDontApplySet(this, op);
+        }
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            collectUnusedAssignedVars((AbstractLogicalOperator) c.getValue(), toRemove, false, context);
+        }
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+                    collectUnusedAssignedVars((AbstractLogicalOperator) r.getValue(), toRemove, false, context);
+                }
+            }
+        }
+        boolean removeUsedVars = true;
+        switch (op.getOperatorTag()) {
+            case ASSIGN: {
+                AssignOperator assign = (AssignOperator) op;
+                toRemove.addAll(assign.getVariables());
+                break;
+            }
+            case AGGREGATE: {
+                AggregateOperator agg = (AggregateOperator) op;
+                toRemove.addAll(agg.getVariables());
+                break;
+            }
+            case UNNEST: {
+                UnnestOperator uOp = (UnnestOperator) op;
+                LogicalVariable pVar = uOp.getPositionalVariable();
+                if (pVar != null) {
+                    toRemove.add(pVar);
+                }
+                break;
+            }
+            case UNIONALL: {
+                UnionAllOperator unionOp = (UnionAllOperator) op;
+                for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : unionOp
+                        .getVariableMappings()) {
+                    toRemove.add(varMapping.third);
+                }
+                removeUsedVars = false;
+                break;
+            }
+        }
+        if (removeUsedVars) {
+            List<LogicalVariable> used = new LinkedList<LogicalVariable>();
+            VariableUtilities.getUsedVariables(op, used);
+            toRemove.removeAll(used);
+        }
+    }
+
+}

[28/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
new file mode 100644
index 0000000..695078a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -0,0 +1,469 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SubstituteVariableVisitor implements ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> {
+
+    private final boolean goThroughNts;
+    private final ITypingContext ctx;
+
+    public SubstituteVariableVisitor(boolean goThroughNts, ITypingContext ctx) {
+        this.goThroughNts = goThroughNts;
+        this.ctx = ctx;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            if (variables.get(i).equals(pair.first)) {
+                variables.set(i, pair.second);
+            } else {
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            if (variables.get(i).equals(pair.first)) {
+                variables.set(i, pair.second);
+            } else {
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+            }
+        }
+        // Substitute variables stored in ordering property
+        if (op.getExplicitOrderingProperty() != null) {
+            List<OrderColumn> orderColumns = op.getExplicitOrderingProperty().getOrderColumns();
+            for (int i = 0; i < orderColumns.size(); i++) {
+                OrderColumn oc = orderColumns.get(i);
+                if (oc.getColumn().equals(pair.first)) {
+                    orderColumns.set(i, new OrderColumn(pair.second, oc.getOrder()));
+                }
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+            eRef.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        subst(pair.first, pair.second, op.getGroupByList());
+        subst(pair.first, pair.second, op.getDecorList());
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
+                        pair.second, goThroughNts, ctx);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
+        ILogicalExpression offset = op.getOffset().getValue();
+        if (offset != null) {
+            offset.substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+            oe.second.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> usedVariables = op.getVariables();
+        int n = usedVariables.size();
+        for (int i = 0; i < n; i++) {
+            LogicalVariable v = usedVariables.get(i);
+            if (v.equals(pair.first)) {
+                usedVariables.set(i, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            if (variables.get(i).equals(pair.first)) {
+                variables.set(i, pair.second);
+            } else {
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        substInArray(op.getInputVariables(), pair.first, pair.second);
+        substInArray(op.getOutputVariables(), pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
+                        pair.second, goThroughNts, ctx);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+            if (t.first.equals(pair.first)) {
+                t.first = pair.second;
+            }
+            if (t.second.equals(pair.first)) {
+                t.second = pair.second;
+            }
+            if (t.third.equals(pair.first)) {
+                t.third = pair.second;
+            }
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    private void subst(LogicalVariable v1, LogicalVariable v2,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> varExprPairList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : varExprPairList) {
+            if (ve.first != null && ve.first.equals(v1)) {
+                ve.first = v2;
+                return;
+            }
+            ve.second.getValue().substituteVar(v1, v2);
+        }
+    }
+
+    private void substInArray(ArrayList<LogicalVariable> varArray, LogicalVariable v1, LogicalVariable v2) {
+        for (int i = 0; i < varArray.size(); i++) {
+            LogicalVariable v = varArray.get(i);
+            if (v == v1) {
+                varArray.set(i, v2);
+            }
+        }
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        op.substituteVar(arg.first, arg.second);
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
+        }
+        substVarTypes(op, pair);
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+            throws AlgebricksException {
+        return null;
+    }
+
+    private void substVarTypes(ILogicalOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        if (ctx == null) {
+            return;
+        }
+        IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(op);
+        env.substituteProducedVariable(arg.first, arg.second);
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op,
+            Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException {
+        List<LogicalVariable> variables = op.getVariables();
+        for (int i = 0; i < variables.size(); i++) {
+            if (variables.get(i) == pair.first) {
+                variables.set(i, pair.second);
+                return null;
+            }
+        }
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+        substVarTypes(op, pair);
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
new file mode 100644
index 0000000..8169ad0
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -0,0 +1,407 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionMergePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+    private Collection<LogicalVariable> usedVariables;
+
+    public UsedVariableVisitor(Collection<LogicalVariable> usedVariables) {
+        this.usedVariables = usedVariables;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) {
+        if (op.getAdditionalFilteringExpressions() != null) {
+            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+                e.getValue().getUsedVariables(usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+            eRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        // Used variables depend on the physical operator.
+        if (op.getPhysicalOperator() != null) {
+            IPhysicalOperator physOp = op.getPhysicalOperator();
+            switch (physOp.getOperatorTag()) {
+                case BROADCAST_EXCHANGE:
+                case ONE_TO_ONE_EXCHANGE:
+                case RANDOM_MERGE_EXCHANGE: {
+                    // No variables used.
+                    break;
+                }
+                case HASH_PARTITION_EXCHANGE: {
+                    HashPartitionExchangePOperator concreteOp = (HashPartitionExchangePOperator) physOp;
+                    usedVariables.addAll(concreteOp.getHashFields());
+                    break;
+                }
+                case HASH_PARTITION_MERGE_EXCHANGE: {
+                    HashPartitionMergeExchangePOperator concreteOp = (HashPartitionMergeExchangePOperator) physOp;
+                    usedVariables.addAll(concreteOp.getPartitionFields());
+                    for (OrderColumn orderCol : concreteOp.getOrderColumns()) {
+                        usedVariables.add(orderCol.getColumn());
+                    }
+                    break;
+                }
+                case SORT_MERGE_EXCHANGE: {
+                    SortMergeExchangePOperator concreteOp = (SortMergeExchangePOperator) physOp;
+                    for (OrderColumn orderCol : concreteOp.getSortColumns()) {
+                        usedVariables.add(orderCol.getColumn());
+                    }
+                    break;
+                }
+                case RANGE_PARTITION_EXCHANGE: {
+                    RangePartitionPOperator concreteOp = (RangePartitionPOperator) physOp;
+                    for (OrderColumn partCol : concreteOp.getPartitioningFields()) {
+                        usedVariables.add(partCol.getColumn());
+                    }
+                    break;
+                }
+                case RANGE_PARTITION_MERGE_EXCHANGE: {
+                    RangePartitionMergePOperator concreteOp = (RangePartitionMergePOperator) physOp;
+                    for (OrderColumn partCol : concreteOp.getPartitioningFields()) {
+                        usedVariables.add(partCol.getColumn());
+                    }
+                    break;
+                }
+                default: {
+                    throw new AlgebricksException("Unhandled physical operator tag '" + physOp.getOperatorTag() + "'.");
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) {
+            g.second.getValue().getUsedVariables(usedVariables);
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) {
+            g.second.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) {
+        op.getCondition().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) {
+        op.getCondition().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Void arg) {
+        op.getMaxObjects().getValue().getUsedVariables(usedVariables);
+        ILogicalExpression offsetExpr = op.getOffset().getValue();
+        if (offsetExpr != null) {
+            offsetExpr.getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) {
+        // does not use any variable
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Void arg) {
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+            oe.second.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Void arg) {
+        List<LogicalVariable> parameterVariables = op.getVariables();
+        for (LogicalVariable v : parameterVariables) {
+            if (!usedVariables.contains(v)) {
+                usedVariables.add(v);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Void arg) {
+        List<LogicalVariable> parameterVariables = op.getInputVariables();
+        for (LogicalVariable v : parameterVariables) {
+            if (!usedVariables.contains(v)) {
+                usedVariables.add(v);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Void arg) {
+        op.getCondition().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Void arg) {
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> m : op.getVariableMappings()) {
+            if (!usedVariables.contains(m.first)) {
+                usedVariables.add(m.first);
+            }
+            if (!usedVariables.contains(m.second)) {
+                usedVariables.add(m.second);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        if (op.getAdditionalFilteringExpressions() != null) {
+            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+                e.getValue().getUsedVariables(usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Void arg) {
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
+            expr.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
+            expr.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) {
+        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        if (op.getAdditionalFilteringExpressions() != null) {
+            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+                e.getValue().getUsedVariables(usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) {
+        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        if (op.getAdditionalFilteringExpressions() != null) {
+            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+                e.getValue().getUsedVariables(usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        if (op.getAdditionalFilteringExpressions() != null) {
+            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
+                e.getValue().getUsedVariables(usedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, Void arg) {
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Void arg) {
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        op.getDelegate().getUsedVariables(usedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
new file mode 100644
index 0000000..f3cf0a4
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class VariableUtilities {
+
+    public static void getUsedVariables(ILogicalOperator op, Collection<LogicalVariable> usedVariables)
+            throws AlgebricksException {
+        ILogicalOperatorVisitor<Void, Void> visitor = new UsedVariableVisitor(usedVariables);
+        op.accept(visitor, null);
+    }
+
+    public static void getProducedVariables(ILogicalOperator op, Collection<LogicalVariable> producedVariables)
+            throws AlgebricksException {
+        ILogicalOperatorVisitor<Void, Void> visitor = new ProducedVariableVisitor(producedVariables);
+        op.accept(visitor, null);
+    }
+
+    public static void getLiveVariables(ILogicalOperator op, Collection<LogicalVariable> schemaVariables)
+            throws AlgebricksException {
+        ILogicalOperatorVisitor<Void, Void> visitor = new SchemaVariableVisitor(schemaVariables);
+        op.accept(visitor, null);
+    }
+
+    public static void getUsedVariablesInDescendantsAndSelf(ILogicalOperator op, Collection<LogicalVariable> vars)
+            throws AlgebricksException {
+        // DFS traversal
+        VariableUtilities.getUsedVariables(op, vars);
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            getUsedVariablesInDescendantsAndSelf(c.getValue(), vars);
+        }
+    }
+
+    public static void getProducedVariablesInDescendantsAndSelf(ILogicalOperator op, Collection<LogicalVariable> vars)
+            throws AlgebricksException {
+        // DFS traversal
+        VariableUtilities.getProducedVariables(op, vars);
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            getProducedVariablesInDescendantsAndSelf(c.getValue(), vars);
+        }
+    }
+
+    public static void substituteVariables(ILogicalOperator op, LogicalVariable v1, LogicalVariable v2,
+            ITypingContext ctx) throws AlgebricksException {
+        substituteVariables(op, v1, v2, true, ctx);
+    }
+
+    public static void substituteVariablesInDescendantsAndSelf(ILogicalOperator op, LogicalVariable v1,
+            LogicalVariable v2, ITypingContext ctx) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> childOp : op.getInputs()) {
+            substituteVariablesInDescendantsAndSelf(childOp.getValue(), v1, v2, ctx);
+        }
+        substituteVariables(op, v1, v2, true, ctx);
+    }
+
+    public static void substituteVariables(ILogicalOperator op, LogicalVariable v1, LogicalVariable v2,
+            boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
+        ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> visitor = new SubstituteVariableVisitor(
+                goThroughNts, ctx);
+        op.accept(visitor, new Pair<LogicalVariable, LogicalVariable>(v1, v2));
+    }
+
+    public static <T> boolean varListEqualUnordered(List<T> var, List<T> varArg) {
+        Set<T> varSet = new HashSet<T>();
+        Set<T> varArgSet = new HashSet<T>();
+        varSet.addAll(var);
+        varArgSet.addAll(varArg);
+        return varSet.equals(varArgSet);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
new file mode 100644
index 0000000..42d964d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+
+public abstract class AbstractExchangePOperator extends AbstractPhysicalOperator {
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op,
+                opSchema, context);
+        builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+
+    public abstract Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(
+            IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
new file mode 100644
index 0000000..764159d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+
+public abstract class AbstractHashJoinPOperator extends AbstractJoinPOperator {
+
+    protected List<LogicalVariable> keysLeftBranch;
+    protected List<LogicalVariable> keysRightBranch;
+
+    public AbstractHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
+            List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities) {
+        super(kind, partitioningType);
+        this.keysLeftBranch = sideLeftOfEqualities;
+        this.keysRightBranch = sideRightOfEqualities;
+    }
+
+    public List<LogicalVariable> getKeysLeftBranch() {
+        return keysLeftBranch;
+    }
+
+    public List<LogicalVariable> getKeysRightBranch() {
+        return keysRightBranch;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context)
+            throws AlgebricksException {
+        IPartitioningProperty pp;
+        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+
+        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+            AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+            IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
+            AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
+            IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties();
+
+            if (pv0 == null || pv1 == null) {
+                pp = null;
+            } else {
+                pp = pv0.getPartitioningProperty();
+            }
+        } else {
+            pp = IPartitioningProperty.UNPARTITIONED;
+        }
+        this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context));
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop,
+            IPhysicalPropertiesVector reqdByParent) {
+        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
+        // In a cost-based optimizer, we would also try to propagate the
+        // parent's partitioning requirements.
+        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+
+        IPartitioningProperty pp1 = null;
+        IPartitioningProperty pp2 = null;
+        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+            switch (partitioningType) {
+                case PAIRWISE: {
+                    pp1 = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(keysLeftBranch), null);
+                    pp2 = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(keysRightBranch), null);
+                    break;
+                }
+                case BROADCAST: {
+                    pp2 = new BroadcastPartitioningProperty(null);
+                    break;
+                }
+                default: {
+                    throw new IllegalStateException();
+                }
+            }
+        }
+
+        pv[0] = new StructuralPropertiesVector(pp1, null);
+        pv[1] = new StructuralPropertiesVector(pp2, null);
+
+        IPartitioningRequirementsCoordinator prc;
+        switch (kind) {
+            case INNER: {
+                prc = IPartitioningRequirementsCoordinator.EQCLASS_PARTITIONING_COORDINATOR;
+                break;
+            }
+            case LEFT_OUTER: {
+                prc = new IPartitioningRequirementsCoordinator() {
+
+                    @Override
+                    public Pair<Boolean, IPartitioningProperty> coordinateRequirements(
+                            IPartitioningProperty requirements, IPartitioningProperty firstDeliveredPartitioning,
+                            ILogicalOperator op, IOptimizationContext context) throws AlgebricksException {
+                        if (firstDeliveredPartitioning != null
+                                && firstDeliveredPartitioning.getPartitioningType() == requirements
+                                        .getPartitioningType()) {
+                            switch (requirements.getPartitioningType()) {
+                                case UNORDERED_PARTITIONED: {
+                                    UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
+                                    Set<LogicalVariable> set1 = upp1.getColumnSet();
+                                    UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) requirements;
+                                    Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
+                                    Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
+                                    Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
+                                    for (LogicalVariable r : uppreq.getColumnSet()) {
+                                        EquivalenceClass ecSnd = eqmap.get(r);
+                                        boolean found = false;
+                                        int j = 0;
+                                        for (LogicalVariable rvar : keysRightBranch) {
+                                            if (rvar == r || ecSnd != null && eqmap.get(rvar) == ecSnd) {
+                                                found = true;
+                                                break;
+                                            }
+                                            j++;
+                                        }
+                                        if (!found) {
+                                            throw new IllegalStateException("Did not find a variable equivalent to "
+                                                    + r + " among " + keysRightBranch);
+                                        }
+                                        LogicalVariable v2 = keysLeftBranch.get(j);
+                                        EquivalenceClass ecFst = eqmap.get(v2);
+                                        for (LogicalVariable vset1 : set1) {
+                                            if (vset1 == v2 || ecFst != null && eqmap.get(vset1) == ecFst) {
+                                                covered.add(vset1);
+                                                modifuppreq.add(r);
+                                                break;
+                                            }
+                                        }
+                                    }
+                                    if (!covered.equals(set1)) {
+                                        throw new AlgebricksException("Could not modify " + requirements
+                                                + " to agree with partitioning property " + firstDeliveredPartitioning
+                                                + " delivered by previous input operator.");
+                                    }
+                                    UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq,
+                                            requirements.getNodeDomain());
+                                    return new Pair<Boolean, IPartitioningProperty>(false, upp2);
+                                }
+                                case ORDERED_PARTITIONED: {
+                                    throw new NotImplementedException();
+                                }
+                            }
+                        }
+                        return new Pair<Boolean, IPartitioningProperty>(true, requirements);
+                    }
+                };
+                break;
+            }
+            default: {
+                throw new IllegalStateException();
+            }
+        }
+
+        return new PhysicalRequirements(pv, prc);
+    }
+
+    protected abstract List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op,
+            IOptimizationContext context) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
new file mode 100644
index 0000000..fcc04ab
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+
+public abstract class AbstractJoinPOperator extends AbstractPhysicalOperator {
+
+    public enum JoinPartitioningType {
+        PAIRWISE, BROADCAST
+    }
+
+    protected final JoinKind kind;
+    protected final JoinPartitioningType partitioningType;
+
+    public AbstractJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType) {
+        this.kind = kind;
+        this.partitioningType = partitioningType;
+    }
+
+    public JoinKind getKind() {
+        return kind;
+    }
+
+    public JoinPartitioningType getPartitioningType() {
+        return partitioningType;
+    }
+
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[] { 1, 0 };
+        int[] outputDependencyLabels = new int[] { 1 };
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
new file mode 100644
index 0000000..66e7b98
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public abstract class AbstractPhysicalOperator implements IPhysicalOperator {
+
+    protected IPhysicalPropertiesVector deliveredProperties;
+    private boolean disableJobGenBelow = false;
+    private Object hostQueryContext;
+
+    @Override
+    public final IPhysicalPropertiesVector getDeliveredProperties() {
+        return deliveredProperties;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString();
+    }
+
+    public void setHostQueryContext(Object context) {
+        this.hostQueryContext = context;
+    }
+
+    public Object getHostQueryContext() {
+        return hostQueryContext;
+    }
+
+    protected PhysicalRequirements emptyUnaryRequirements() {
+        StructuralPropertiesVector[] req = new StructuralPropertiesVector[] { StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR };
+        return new PhysicalRequirements(req, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    protected PhysicalRequirements emptyUnaryRequirements(int numberOfChildren) {
+        StructuralPropertiesVector[] req = new StructuralPropertiesVector[numberOfChildren];
+        for (int i = 0; i < numberOfChildren; i++) {
+            req[i] = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
+        }
+        return new PhysicalRequirements(req, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void disableJobGenBelowMe() {
+        this.disableJobGenBelow = true;
+    }
+
+    @Override
+    public boolean isJobGenDisabledBelowMe() {
+        return disableJobGenBelow;
+    }
+
+    /**
+     * @return labels (0 or 1) for each input and output indicating the dependency between them.
+     *         The edges labeled as 1 must wait for the edges with label 0.
+     */
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[op.getInputs().size()]; // filled with 0's
+        int[] outputDependencyLabels = new int[] { 0 };
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    protected void contributeOpDesc(IHyracksJobBuilder builder, AbstractLogicalOperator op, IOperatorDescriptor opDesc) {
+        if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
+            AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1);
+            builder.contributeAlgebricksPartitionConstraint(opDesc, apc);
+        }
+        builder.contributeHyracksOperator(op, opDesc);
+    }
+
+    protected AlgebricksPipeline[] compileSubplans(IOperatorSchema outerPlanSchema,
+            AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, JobGenContext context)
+            throws AlgebricksException {
+        AlgebricksPipeline[] subplans = new AlgebricksPipeline[npOp.getNestedPlans().size()];
+        PlanCompiler pc = new PlanCompiler(context);
+        int i = 0;
+        for (ILogicalPlan p : npOp.getNestedPlans()) {
+            subplans[i++] = buildPipelineWithProjection(p, outerPlanSchema, npOp, opSchema, pc);
+        }
+        return subplans;
+    }
+
+    private AlgebricksPipeline buildPipelineWithProjection(ILogicalPlan p, IOperatorSchema outerPlanSchema,
+            AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, PlanCompiler pc) throws AlgebricksException {
+        if (p.getRoots().size() > 1) {
+            throw new NotImplementedException("Nested plans with several roots are not supported.");
+        }
+        JobSpecification nestedJob = pc.compilePlan(p, outerPlanSchema, null);
+        ILogicalOperator topOpInSubplan = p.getRoots().get(0).getValue();
+        JobGenContext context = pc.getContext();
+        IOperatorSchema topOpInSubplanScm = context.getSchema(topOpInSubplan);
+        opSchema.addAllVariables(topOpInSubplanScm);
+
+        Map<OperatorDescriptorId, IOperatorDescriptor> opMap = nestedJob.getOperatorMap();
+        if (opMap.size() != 1) {
+            throw new AlgebricksException(
+                    "Attempting to construct a nested plan with "
+                            + opMap.size()
+                            + " operator descriptors. Currently, nested plans can only consist in linear pipelines of Asterix micro operators.");
+        }
+
+        for (OperatorDescriptorId oid : opMap.keySet()) {
+            IOperatorDescriptor opd = opMap.get(oid);
+            if (!(opd instanceof AlgebricksMetaOperatorDescriptor)) {
+                throw new AlgebricksException(
+                        "Can only generate Hyracks jobs for pipelinable Asterix nested plans, not for "
+                                + opd.getClass().getName());
+            }
+            AlgebricksMetaOperatorDescriptor amod = (AlgebricksMetaOperatorDescriptor) opd;
+
+            return amod.getPipeline();
+            // we suppose that the top operator in the subplan already does the
+            // projection for us
+        }
+
+        throw new IllegalStateException();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
new file mode 100644
index 0000000..41711cb
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
@@ -0,0 +1,265 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+
+public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysicalOperator {
+
+    protected List<LogicalVariable> columnList;
+
+    public AbstractPreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
+        this.columnList = columnList;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + columnList;
+    }
+
+    public List<LogicalVariable> getGbyColumns() {
+        return columnList;
+    }
+
+    public void setGbyColumns(List<LogicalVariable> gByColumns) {
+        this.columnList = gByColumns;
+    }
+
+    // Obs: We don't propagate properties corresponding to decors, since they
+    // are func. dep. on the group-by variables.
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
+        GroupByOperator gby = (GroupByOperator) op;
+        ILogicalOperator op2 = gby.getInputs().get(0).getValue();
+        IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
+        IPartitioningProperty pp = childProp.getPartitioningProperty();
+        List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties();
+        if (childLocals != null) {
+            for (ILocalStructuralProperty lsp : childLocals) {
+                boolean failed = false;
+                switch (lsp.getPropertyType()) {
+                    case LOCAL_GROUPING_PROPERTY: {
+                        LocalGroupingProperty lgp = (LocalGroupingProperty) lsp;
+                        Set<LogicalVariable> colSet = new ListSet<LogicalVariable>();
+                        for (LogicalVariable v : lgp.getColumnSet()) {
+                            LogicalVariable v2 = getLhsGbyVar(gby, v);
+                            if (v2 != null) {
+                                colSet.add(v2);
+                            } else {
+                                failed = true;
+                            }
+                        }
+                        if (!failed) {
+                            propsLocal.add(new LocalGroupingProperty(colSet));
+                        }
+                        break;
+                    }
+                    case LOCAL_ORDER_PROPERTY: {
+                        LocalOrderProperty lop = (LocalOrderProperty) lsp;
+                        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+                        for (OrderColumn oc : lop.getOrderColumns()) {
+                            LogicalVariable v2 = getLhsGbyVar(gby, oc.getColumn());
+                            if (v2 != null) {
+                                orderColumns.add(new OrderColumn(v2, oc.getOrder()));
+                            } else {
+                                failed = true;
+                            }
+                        }
+                        if (!failed) {
+                            propsLocal.add(new LocalOrderProperty(orderColumns));
+                        }
+                        break;
+                    }
+                    default: {
+                        throw new IllegalStateException();
+                    }
+                }
+                if (failed) {
+                    break;
+                }
+            }
+        }
+        deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+        List<ILocalStructuralProperty> localProps = null;
+
+        localProps = new ArrayList<ILocalStructuralProperty>(1);
+        Set<LogicalVariable> gbvars = new ListSet<LogicalVariable>(columnList);
+        LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<LogicalVariable>(columnList));
+
+        GroupByOperator gby = (GroupByOperator) op;
+        boolean goon = true;
+        for (ILogicalPlan p : gby.getNestedPlans()) {
+            // try to propagate secondary order requirements from nested
+            // groupings
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
+                if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+                    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+                    IPhysicalOperator pop2 = op2.getPhysicalOperator();
+                    if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
+                        List<LogicalVariable> sndOrder = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
+                        groupProp.getColumnSet().addAll(sndOrder);
+                        groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
+                        goon = false;
+                        break;
+                    }
+                }
+            }
+            if (!goon) {
+                break;
+            }
+        }
+
+        localProps.add(groupProp);
+
+        if (reqdByParent != null) {
+            // propagate parent requirements
+            List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
+            if (lpPar != null) {
+                boolean allOk = true;
+                List<ILocalStructuralProperty> props = new ArrayList<ILocalStructuralProperty>(lpPar.size());
+                for (ILocalStructuralProperty prop : lpPar) {
+                    if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+                        allOk = false;
+                        break;
+                    }
+                    LocalOrderProperty lop = (LocalOrderProperty) prop;
+                    List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+                    List<OrderColumn> ords = lop.getOrderColumns();
+                    for (OrderColumn ord : ords) {
+                        Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
+                        if (p == null) {
+                            p = getDecorPairByRhsVar(gby, ord.getColumn());
+                            if (p == null) {
+                                allOk = false;
+                                break;
+                            }
+                        }
+                        ILogicalExpression e = p.second.getValue();
+                        if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                            throw new IllegalStateException(
+                                    "Right hand side of group-by assignment should have been normalized to a variable reference.");
+                        }
+                        LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
+                        orderColumns.add(new OrderColumn(v, ord.getOrder()));
+                    }
+                    props.add(new LocalOrderProperty(orderColumns));
+                }
+                List<FunctionalDependency> fdList = new ArrayList<FunctionalDependency>();
+                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
+                    List<LogicalVariable> hd = gby.getGbyVarList();
+                    List<LogicalVariable> tl = new ArrayList<LogicalVariable>(1);
+                    tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
+                    fdList.add(new FunctionalDependency(hd, tl));
+                }
+                if (allOk
+                        && PropertiesUtil.matchLocalProperties(localProps, props,
+                                new HashMap<LogicalVariable, EquivalenceClass>(), fdList)) {
+                    localProps = props;
+                }
+            }
+        }
+
+        IPartitioningProperty pp = null;
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
+        if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
+            pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), null);
+        }
+        pv[0] = new StructuralPropertiesVector(pp, localProps);
+        return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getGbyPairByRhsVar(GroupByOperator gby,
+            LogicalVariable var) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
+            if (ve.first == var) {
+                return ve;
+            }
+        }
+        return null;
+    }
+
+    private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getDecorPairByRhsVar(GroupByOperator gby,
+            LogicalVariable var) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getDecorList()) {
+            if (ve.first == var) {
+                return ve;
+            }
+        }
+        return null;
+    }
+
+    private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable var) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
+            ILogicalExpression e = ve.second.getValue();
+            if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new IllegalStateException(
+                        "Right hand side of group by assignment should have been normalized to a variable reference.");
+            }
+            LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
+            if (v == var) {
+                return ve.first;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
new file mode 100644
index 0000000..c77222b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+
+public abstract class AbstractPropagatePropertiesForUsedVariablesPOperator extends AbstractPhysicalOperator {
+
+    public void computeDeliveredPropertiesForUsedVariables(ILogicalOperator op, List<LogicalVariable> usedVariables) {
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
+        List<ILocalStructuralProperty> downPropsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
+        List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
+        for (ILocalStructuralProperty lsp : downPropsLocal) {
+            LinkedList<LogicalVariable> cols = new LinkedList<LogicalVariable>();
+            lsp.getColumns(cols);
+            ILocalStructuralProperty propagatedProp = lsp.retainVariables(usedVariables);
+            if (propagatedProp != null) {
+                propsLocal.add(propagatedProp);
+            }
+        }
+        deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+    }
+}

[35/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
deleted file mode 100644
index 3af57ad..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class PlanCompiler {
-    private JobGenContext context;
-    private Map<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> operatorVisitedToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
-
-    public PlanCompiler(JobGenContext context) {
-        this.context = context;
-    }
-
-    public JobGenContext getContext() {
-        return context;
-    }
-
-    public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema,
-            IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException {
-        JobSpecification spec = new JobSpecification(context.getFrameSize());
-        if (jobEventListenerFactory != null) {
-            spec.setJobletEventListenerFactory(jobEventListenerFactory);
-        }
-        List<ILogicalOperator> rootOps = new ArrayList<ILogicalOperator>();
-        IHyracksJobBuilder builder = new JobBuilder(spec, context.getClusterLocations());
-        for (Mutable<ILogicalOperator> opRef : plan.getRoots()) {
-            compileOpRef(opRef, spec, builder, outerPlanSchema);
-            rootOps.add(opRef.getValue());
-        }
-        reviseEdges(builder);
-        operatorVisitedToParents.clear();
-        builder.buildSpec(rootOps);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
-        // Do not do activity cluster planning because it is slow on large clusters
-        spec.setUseConnectorPolicyForScheduling(false);
-        return spec;
-    }
-
-    private void compileOpRef(Mutable<ILogicalOperator> opRef, IOperatorDescriptorRegistry spec,
-            IHyracksJobBuilder builder, IOperatorSchema outerPlanSchema) throws AlgebricksException {
-        ILogicalOperator op = opRef.getValue();
-        int n = op.getInputs().size();
-        IOperatorSchema[] schemas = new IOperatorSchema[n];
-        int i = 0;
-        for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
-            List<Mutable<ILogicalOperator>> parents = operatorVisitedToParents.get(opRef2);
-            if (parents == null) {
-                parents = new ArrayList<Mutable<ILogicalOperator>>();
-                operatorVisitedToParents.put(opRef2, parents);
-                parents.add(opRef);
-                compileOpRef(opRef2, spec, builder, outerPlanSchema);
-                schemas[i++] = context.getSchema(opRef2.getValue());
-            } else {
-                if (!parents.contains(opRef))
-                    parents.add(opRef);
-                schemas[i++] = context.getSchema(opRef2.getValue());
-                continue;
-            }
-        }
-
-        IOperatorSchema opSchema = new OperatorSchemaImpl();
-        context.putSchema(op, opSchema);
-        op.getVariablePropagationPolicy().propagateVariables(opSchema, schemas);
-        op.contributeRuntimeOperator(builder, context, opSchema, schemas, outerPlanSchema);
-    }
-
-    private void reviseEdges(IHyracksJobBuilder builder) {
-        /**
-         * revise the edges for the case of replicate operator
-         */
-        for (Entry<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> entry : operatorVisitedToParents
-                .entrySet()) {
-            Mutable<ILogicalOperator> child = entry.getKey();
-            List<Mutable<ILogicalOperator>> parents = entry.getValue();
-            if (parents.size() > 1) {
-                if (child.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) {
-                    ReplicateOperator rop = (ReplicateOperator) child.getValue();
-                    if (rop.isBlocker()) {
-                        // make the order of the graph edges consistent with the order of rop's outputs
-                        List<Mutable<ILogicalOperator>> outputs = rop.getOutputs();
-                        for (Mutable<ILogicalOperator> parent : parents) {
-                            builder.contributeGraphEdge(child.getValue(), outputs.indexOf(parent), parent.getValue(), 0);
-                        }
-                    } else {
-                        int i = 0;
-                        for (Mutable<ILogicalOperator> parent : parents) {
-                            builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0);
-                            i++;
-                        }
-                    }
-                }
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
deleted file mode 100644
index e7b469a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
-
-import java.util.Collection;
-import java.util.logging.Level;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-
-public abstract class AbstractRuleController {
-
-    protected IOptimizationContext context;
-
-    public AbstractRuleController() {
-    }
-
-    public void setContext(IOptimizationContext context) {
-        this.context = context;
-    }
-
-    /**
-     * Each rewriting strategy may differ in the
-     * 
-     * @param root
-     * @param ruleClasses
-     * @return true iff one of the rules in the collection fired
-     */
-    public abstract boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
-            Collection<IAlgebraicRewriteRule> rules) throws AlgebricksException;
-
-    /**
-     * @param opRef
-     * @param rule
-     * @return true if any rewrite was fired, either on opRef or any operator
-     *         under it.
-     */
-    protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule)
-            throws AlgebricksException {
-        return rewriteOperatorRef(opRef, rule, true, false);
-    }
-
-    private String getPlanString(Mutable<ILogicalOperator> opRef) throws AlgebricksException {
-        if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
-            StringBuilder sb = new StringBuilder();
-            LogicalOperatorPrettyPrintVisitor pvisitor = context.getPrettyPrintVisitor();
-            PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
-            return sb.toString();
-        }
-        return null;
-    }
-
-    private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan)
-            throws AlgebricksException {
-        if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Rule " + rule.getClass() + " fired.\n");
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Before plan\n" + beforePlan + "\n");
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> After plan\n" + afterPlan + "\n");
-        }
-    }
-
-    protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule,
-            boolean enterNestedPlans, boolean fullDFS) throws AlgebricksException {
-
-        String preBeforePlan = getPlanString(opRef);
-        if (rule.rewritePre(opRef, context)) {
-            String preAfterPlan = getPlanString(opRef);
-            printRuleApplication(rule, preBeforePlan, preAfterPlan);
-            return true;
-        }
-        boolean rewritten = false;
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-
-        for (Mutable<ILogicalOperator> inp : op.getInputs()) {
-            if (rewriteOperatorRef(inp, rule, enterNestedPlans, fullDFS)) {
-                rewritten = true;
-                if (!fullDFS) {
-                    break;
-                }
-            }
-        }
-
-        if (op.hasNestedPlans() && enterNestedPlans) {
-            AbstractOperatorWithNestedPlans o2 = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : o2.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    if (rewriteOperatorRef(r, rule, enterNestedPlans, fullDFS)) {
-                        rewritten = true;
-                        if (!fullDFS) {
-                            break;
-                        }
-                    }
-                }
-                if (rewritten && !fullDFS) {
-                    break;
-                }
-            }
-        }
-
-        String postBeforePlan = getPlanString(opRef);
-        if (rule.rewritePost(opRef, context)) {
-            String postAfterPlan = getPlanString(opRef);
-            printRuleApplication(rule, postBeforePlan, postAfterPlan);
-            return true;
-        }
-
-        return rewritten;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
deleted file mode 100644
index 361dad3..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
-
-public class AlgebricksOptimizationContext implements IOptimizationContext {
-
-    private int varCounter;
-    private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
-    private final IMergeAggregationExpressionFactory mergeAggregationExpressionFactory;
-    private final PhysicalOptimizationConfig physicalOptimizationConfig;
-    private final IVariableEvalSizeEnvironment varEvalSizeEnv = new IVariableEvalSizeEnvironment() {
-
-        Map<LogicalVariable, Integer> varSizeMap = new HashMap<LogicalVariable, Integer>();
-
-        @Override
-        public void setVariableEvalSize(LogicalVariable var, int size) {
-            varSizeMap.put(var, size);
-        }
-
-        @Override
-        public int getVariableEvalSize(LogicalVariable var) {
-            return varSizeMap.get(var);
-        }
-    };
-
-    private Map<ILogicalOperator, IVariableTypeEnvironment> typeEnvMap = new HashMap<ILogicalOperator, IVariableTypeEnvironment>();
-
-    private Map<ILogicalOperator, HashSet<ILogicalOperator>> alreadyCompared = new HashMap<ILogicalOperator, HashSet<ILogicalOperator>>();
-    private Map<IAlgebraicRewriteRule, HashSet<ILogicalOperator>> dontApply = new HashMap<IAlgebraicRewriteRule, HashSet<ILogicalOperator>>();
-    private Map<LogicalVariable, FunctionalDependency> recordToPrimaryKey = new HashMap<LogicalVariable, FunctionalDependency>();
-
-    @SuppressWarnings("unchecked")
-    private IMetadataProvider metadataProvider;
-    private HashSet<LogicalVariable> notToBeInlinedVars = new HashSet<LogicalVariable>();
-
-    protected final Map<ILogicalOperator, List<FunctionalDependency>> fdGlobalMap = new HashMap<ILogicalOperator, List<FunctionalDependency>>();
-    protected final Map<ILogicalOperator, Map<LogicalVariable, EquivalenceClass>> eqClassGlobalMap = new HashMap<ILogicalOperator, Map<LogicalVariable, EquivalenceClass>>();
-
-    protected final Map<ILogicalOperator, ILogicalPropertiesVector> logicalProps = new HashMap<ILogicalOperator, ILogicalPropertiesVector>();
-    private final IExpressionTypeComputer expressionTypeComputer;
-    private final INullableTypeComputer nullableTypeComputer;
-    private final LogicalOperatorPrettyPrintVisitor prettyPrintVisitor;
-
-    public AlgebricksOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
-            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
-            IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
-            PhysicalOptimizationConfig physicalOptimizationConfig) {
-        this(varCounter, expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
-                nullableTypeComputer, physicalOptimizationConfig, new LogicalOperatorPrettyPrintVisitor());
-    }
-
-    public AlgebricksOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
-            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
-            IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
-            PhysicalOptimizationConfig physicalOptimizationConfig, LogicalOperatorPrettyPrintVisitor prettyPrintVisitor) {
-        this.varCounter = varCounter;
-        this.expressionEvalSizeComputer = expressionEvalSizeComputer;
-        this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
-        this.expressionTypeComputer = expressionTypeComputer;
-        this.nullableTypeComputer = nullableTypeComputer;
-        this.physicalOptimizationConfig = physicalOptimizationConfig;
-        this.prettyPrintVisitor = prettyPrintVisitor;
-    }
-
-    public int getVarCounter() {
-        return varCounter;
-    }
-
-    public void setVarCounter(int varCounter) {
-        this.varCounter = varCounter;
-    }
-
-    public LogicalVariable newVar() {
-        varCounter++;
-        LogicalVariable var = new LogicalVariable(varCounter);
-        return var;
-    }
-
-    @SuppressWarnings("unchecked")
-    public IMetadataProvider getMetadataProvider() {
-        return metadataProvider;
-    }
-
-    public void setMetadataDeclarations(IMetadataProvider<?, ?> metadataProvider) {
-        this.metadataProvider = metadataProvider;
-    }
-
-    public boolean checkIfInDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op) {
-        HashSet<ILogicalOperator> operators = dontApply.get(rule);
-        if (operators == null) {
-            return false;
-        } else {
-            return operators.contains(op);
-        }
-    }
-
-    public void addToDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op) {
-        HashSet<ILogicalOperator> operators = dontApply.get(rule);
-        if (operators == null) {
-            HashSet<ILogicalOperator> os = new HashSet<ILogicalOperator>();
-            os.add(op);
-            dontApply.put(rule, os);
-        } else {
-            operators.add(op);
-        }
-
-    }
-
-    /*
-     * returns true if op1 and op2 have already been compared
-     */
-    @Override
-    public boolean checkAndAddToAlreadyCompared(ILogicalOperator op1, ILogicalOperator op2) {
-        HashSet<ILogicalOperator> ops = alreadyCompared.get(op1);
-        if (ops == null) {
-            HashSet<ILogicalOperator> newEntry = new HashSet<ILogicalOperator>();
-            newEntry.add(op2);
-            alreadyCompared.put(op1, newEntry);
-            return false;
-        } else {
-            if (ops.contains(op2)) {
-                return true;
-            } else {
-                ops.add(op2);
-                return false;
-            }
-        }
-    }
-    
-    @Override
-    public void removeFromAlreadyCompared(ILogicalOperator op1) {
-        alreadyCompared.remove(op1);
-    }
-
-    public void addNotToBeInlinedVar(LogicalVariable var) {
-        notToBeInlinedVars.add(var);
-    }
-
-    public boolean shouldNotBeInlined(LogicalVariable var) {
-        return notToBeInlinedVars.contains(var);
-    }
-
-    public void addPrimaryKey(FunctionalDependency pk) {
-        assert (pk.getTail().size() == 1);
-        LogicalVariable recordVar = pk.getTail().get(0);
-        recordToPrimaryKey.put(recordVar, pk);
-    }
-
-    public List<LogicalVariable> findPrimaryKey(LogicalVariable recordVar) {
-        FunctionalDependency fd = recordToPrimaryKey.get(recordVar);
-        if (fd == null) {
-            return null;
-        }
-        return fd.getHead();
-    }
-
-    @Override
-    public Map<LogicalVariable, EquivalenceClass> getEquivalenceClassMap(ILogicalOperator op) {
-        return eqClassGlobalMap.get(op);
-    }
-
-    @Override
-    public List<FunctionalDependency> getFDList(ILogicalOperator op) {
-        return fdGlobalMap.get(op);
-    }
-
-    @Override
-    public void putEquivalenceClassMap(ILogicalOperator op, Map<LogicalVariable, EquivalenceClass> eqClassMap) {
-        this.eqClassGlobalMap.put(op, eqClassMap);
-    }
-
-    @Override
-    public void putFDList(ILogicalOperator op, List<FunctionalDependency> fdList) {
-        this.fdGlobalMap.put(op, fdList);
-    }
-
-    @Override
-    public ILogicalPropertiesVector getLogicalPropertiesVector(ILogicalOperator op) {
-        return logicalProps.get(op);
-    }
-
-    @Override
-    public void putLogicalPropertiesVector(ILogicalOperator op, ILogicalPropertiesVector v) {
-        logicalProps.put(op, v);
-    }
-
-    @Override
-    public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
-        return expressionEvalSizeComputer;
-    }
-
-    @Override
-    public IVariableEvalSizeEnvironment getVariableEvalSizeEnvironment() {
-        return varEvalSizeEnv;
-    }
-
-    public IMergeAggregationExpressionFactory getMergeAggregationExpressionFactory() {
-        return mergeAggregationExpressionFactory;
-    }
-
-    public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
-        return physicalOptimizationConfig;
-    }
-
-    @Override
-    public IVariableTypeEnvironment getOutputTypeEnvironment(ILogicalOperator op) {
-        return typeEnvMap.get(op);
-    }
-
-    @Override
-    public void setOutputTypeEnvironment(ILogicalOperator op, IVariableTypeEnvironment env) {
-        typeEnvMap.put(op, env);
-    }
-
-    @Override
-    public IExpressionTypeComputer getExpressionTypeComputer() {
-        return expressionTypeComputer;
-    }
-
-    @Override
-    public INullableTypeComputer getNullableTypeComputer() {
-        return nullableTypeComputer;
-    }
-
-    @Override
-    public void invalidateTypeEnvironmentForOperator(ILogicalOperator op) {
-        typeEnvMap.put(op, null);
-    }
-
-    @Override
-    public void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException {
-        setOutputTypeEnvironment(op, op.computeOutputTypeEnvironment(this));
-    }
-
-    @Override
-    public void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars) {
-        for (Map.Entry<LogicalVariable, FunctionalDependency> me : recordToPrimaryKey.entrySet()) {
-            FunctionalDependency fd = me.getValue();
-            List<LogicalVariable> hd = new ArrayList<LogicalVariable>();
-            for (LogicalVariable v : fd.getHead()) {
-                LogicalVariable v2 = mappedVars.get(v);
-                if (v2 == null) {
-                    hd.add(v);
-                } else {
-                    hd.add(v2);
-                }
-            }
-            List<LogicalVariable> tl = new ArrayList<LogicalVariable>();
-            for (LogicalVariable v : fd.getTail()) {
-                LogicalVariable v2 = mappedVars.get(v);
-                if (v2 == null) {
-                    tl.add(v);
-                } else {
-                    tl.add(v2);
-                }
-            }
-            me.setValue(new FunctionalDependency(hd, tl));
-        }
-    }
-    
-    @Override
-    public LogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor() {
-        return prettyPrintVisitor;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
deleted file mode 100644
index 17c6900..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-
-public class HeuristicOptimizer {
-
-    public static PhysicalOperatorTag[] hyraxOperators = new PhysicalOperatorTag[] {
-            PhysicalOperatorTag.DATASOURCE_SCAN, PhysicalOperatorTag.BTREE_SEARCH,
-            PhysicalOperatorTag.EXTERNAL_GROUP_BY, PhysicalOperatorTag.HASH_GROUP_BY, PhysicalOperatorTag.HDFS_READER,
-            PhysicalOperatorTag.HYBRID_HASH_JOIN, PhysicalOperatorTag.IN_MEMORY_HASH_JOIN,
-            PhysicalOperatorTag.NESTED_LOOP, PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY,
-            PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY, PhysicalOperatorTag.SPLIT, PhysicalOperatorTag.STABLE_SORT,
-            PhysicalOperatorTag.UNION_ALL };
-    public static PhysicalOperatorTag[] hyraxOperatorsBelowWhichJobGenIsDisabled = new PhysicalOperatorTag[] {};
-
-    public static boolean isHyraxOp(PhysicalOperatorTag opTag) {
-        for (PhysicalOperatorTag t : hyraxOperators) {
-            if (t == opTag) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private IOptimizationContext context;
-    private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
-    private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
-    private ILogicalPlan plan;
-
-    public HeuristicOptimizer(ILogicalPlan plan,
-            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites,
-            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites,
-            IOptimizationContext context) {
-        this.plan = plan;
-        this.context = context;
-        this.logicalRewrites = logicalRewrites;
-        this.physicalRewrites = physicalRewrites;
-    }
-
-    public void optimize() throws AlgebricksException {
-        if (plan == null) {
-            return;
-        }
-        if (AlgebricksConfig.DEBUG) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting logical optimizations.\n");
-        }
-
-        StringBuilder sb = new StringBuilder();
-        PlanPrettyPrinter.printPlan(plan, sb, context.getPrettyPrintVisitor(), 0);
-        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Logical Plan:\n" + sb.toString());
-        runOptimizationSets(plan, logicalRewrites);
-        computeSchemaBottomUpForPlan(plan);
-        runPhysicalOptimizations(plan, physicalRewrites);
-        StringBuilder sb2 = new StringBuilder();
-        PlanPrettyPrinter.printPlan(plan, sb2, context.getPrettyPrintVisitor(), 0);
-        AlgebricksConfig.ALGEBRICKS_LOGGER.info("Optimized Plan:\n" + sb2.toString());
-    }
-
-    private void runOptimizationSets(ILogicalPlan plan,
-            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> optimSet) throws AlgebricksException {
-        for (Pair<AbstractRuleController, List<IAlgebraicRewriteRule>> ruleList : optimSet) {
-            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
-                ruleList.first.setContext(context);
-                ruleList.first.rewriteWithRuleCollection(r, ruleList.second);
-            }
-        }
-    }
-
-    private static void computeSchemaBottomUpForPlan(ILogicalPlan p) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> r : p.getRoots()) {
-            computeSchemaBottomUpForOp((AbstractLogicalOperator) r.getValue());
-        }
-    }
-
-    private static void computeSchemaBottomUpForOp(AbstractLogicalOperator op) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            computeSchemaBottomUpForOp((AbstractLogicalOperator) i.getValue());
-        }
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : a.getNestedPlans()) {
-                computeSchemaBottomUpForPlan(p);
-            }
-        }
-        op.recomputeSchema();
-    }
-
-    private void runPhysicalOptimizations(ILogicalPlan plan,
-            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites)
-            throws AlgebricksException {
-        if (AlgebricksConfig.DEBUG) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting physical optimizations.\n");
-        }
-        // PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(plan);
-        runOptimizationSets(plan, physicalRewrites);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
deleted file mode 100644
index 6bf0b72..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-
-public interface IAlgebraicRewriteRule {
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException;
-
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
deleted file mode 100644
index 70b7f64..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-
-public interface IOptimizationContextFactory {
-    public IOptimizationContext createOptimizationContext(int varCounter,
-            IExpressionEvalSizeComputer expressionEvalSizeComputer,
-            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
-            IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
-            PhysicalOptimizationConfig physicalOptimizationConfig);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
deleted file mode 100644
index 47d2266..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
-
-import java.util.Properties;
-
-public class PhysicalOptimizationConfig {
-    private static final int MB = 1048576;
-    
-    private static final String FRAMESIZE = "FRAMESIZE";
-    private static final String MAX_FRAMES_EXTERNAL_SORT = "MAX_FRAMES_EXTERNAL_SORT";
-    private static final String MAX_FRAMES_EXTERNAL_GROUP_BY = "MAX_FRAMES_EXTERNAL_GROUP_BY";
-    private static final String MAX_FRAMES_LEFT_INPUT_HYBRID_HASH = "MAX_FRAMES_LEFT_INPUT_HYBRID_HASH";
-    private static final String MAX_FRAMES_HYBRID_HASH = "MAX_FRAMES_HYBRID_HASH";
-    private static final String FUDGE_FACTOR = "FUDGE_FACTOR";
-    private static final String MAX_RECORDS_PER_FRAME = "MAX_RECORDS_PER_FRAME";
-    
-    private static final String DEFAULT_HASH_GROUP_TABLE_SIZE = "DEFAULT_HASH_GROUP_TABLE_SIZE";
-    private static final String DEFAULT_EXTERNAL_GROUP_TABLE_SIZE = "DEFAULT_EXTERNAL_GROUP_TABLE_SIZE";
-    private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
-
-    private Properties properties = new Properties();
-
-    public PhysicalOptimizationConfig() {
-        int frameSize = 32768;
-        setInt(FRAMESIZE, frameSize);
-        setInt(MAX_FRAMES_EXTERNAL_SORT, (int) (((long) 32 * MB) / frameSize));
-        setInt(MAX_FRAMES_EXTERNAL_GROUP_BY, (int) (((long) 32 * MB) / frameSize));
-
-        // use http://www.rsok.com/~jrm/printprimes.html to find prime numbers
-        setInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
-        setInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, 10485767);
-        setInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, 10485767);
-    }
-
-    public int getFrameSize() {
-        return getInt(FRAMESIZE, 32768);
-    }
-
-    public void setFrameSize(int frameSize) {
-        setInt(FRAMESIZE, frameSize);
-    }
-    
-    public double getFudgeFactor() {
-        return getDouble(FUDGE_FACTOR, 1.3);
-    }
-
-    public void setFudgeFactor(double fudgeFactor) {
-        setDouble(FUDGE_FACTOR, fudgeFactor);
-    }
-    
-    public int getMaxRecordsPerFrame() {
-        return getInt(MAX_RECORDS_PER_FRAME, 512);
-    }
-
-    public void setMaxRecordsPerFrame(int maxRecords) {
-        setInt(MAX_RECORDS_PER_FRAME, maxRecords);
-    }
-
-    public int getMaxFramesLeftInputHybridHash() {
-        int frameSize = getFrameSize();
-        return getInt(MAX_FRAMES_LEFT_INPUT_HYBRID_HASH, (int) (140L * 1024 * MB / frameSize));
-    }
-
-    public void setMaxFramesLeftInputHybridHash(int frameLimit) {
-        setInt(MAX_FRAMES_LEFT_INPUT_HYBRID_HASH, frameLimit);
-    }
-    
-    public int getMaxFramesHybridHash() {
-        int frameSize = getFrameSize();
-        return getInt(MAX_FRAMES_HYBRID_HASH, (int) (64L * MB / frameSize));
-    }
-
-    public void setMaxFramesHybridHash(int frameLimit) {
-        setInt(MAX_FRAMES_HYBRID_HASH, frameLimit);
-    }
-
-    public int getMaxFramesExternalGroupBy() {
-        int frameSize = getFrameSize();
-        return getInt(MAX_FRAMES_EXTERNAL_GROUP_BY, (int) (((long) 256 * MB) / frameSize));
-    }
-
-    public void setMaxFramesExternalGroupBy(int frameLimit) {
-        setInt(MAX_FRAMES_EXTERNAL_GROUP_BY, frameLimit);
-    }
-    
-    public int getMaxFramesExternalSort() {
-        int frameSize = getFrameSize();
-        return getInt(MAX_FRAMES_EXTERNAL_SORT, (int) (((long) 32 * MB) / frameSize));
-    }
-
-    public void setMaxFramesExternalSort(int frameLimit) {
-        setInt(MAX_FRAMES_EXTERNAL_SORT, frameLimit);
-    }
-
-    public int getHashGroupByTableSize() {
-        return getInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
-    }
-
-    public void setHashGroupByTableSize(int tableSize) {
-        setInt(DEFAULT_HASH_GROUP_TABLE_SIZE, tableSize);
-    }
-
-    public int getExternalGroupByTableSize() {
-        return getInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, 10485767);
-    }
-
-    public void setExternalGroupByTableSize(int tableSize) {
-        setInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, tableSize);
-    }
-
-    public int getInMemHashJoinTableSize() {
-        return getInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, 10485767);
-    }
-
-    public void setInMemHashJoinTableSize(int tableSize) {
-        setInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, tableSize);
-    }
-
-    private void setInt(String property, int value) {
-        properties.setProperty(property, Integer.toString(value));
-    }
-
-    private int getInt(String property, int defaultValue) {
-        String value = properties.getProperty(property);
-        if (value == null)
-            return defaultValue;
-        else
-            return Integer.parseInt(value);
-    }
-    
-    private void setDouble(String property, double value) {
-        properties.setProperty(property, Double.toString(value));
-    }
-
-    private double getDouble(String property, double defaultValue) {
-        String value = properties.getProperty(property);
-        if (value == null)
-            return defaultValue;
-        else
-            return Double.parseDouble(value);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java
deleted file mode 100644
index 3ef4316..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.utils;
-
-public class Substitution<T> {
-    public T substituted;
-    public T substitutedWith;
-
-    public Substitution(T substituted, T substitutedWith) {
-        this.substituted = substituted;
-        this.substitutedWith = substitutedWith;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/Counter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/Counter.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/Counter.java
new file mode 100644
index 0000000..3dd4704
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/Counter.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public class Counter {
+    private int counter = 0;
+
+    public Counter() {
+    }
+
+    public Counter(int initial) {
+        counter = initial;
+    }
+
+    public int get() {
+        return counter;
+    }
+
+    public void inc() {
+        ++counter;
+    }
+
+    public void set(int newStart) {
+        counter = newStart;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/EquivalenceClass.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
new file mode 100644
index 0000000..ab95c9c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+
+public final class EquivalenceClass {
+    private Set<ILogicalExpression> expressionMembers = new ListSet<ILogicalExpression>();
+    private Set<LogicalVariable> members = new ListSet<LogicalVariable>();
+    private ILogicalExpression constRepresentative;
+    private LogicalVariable variableRepresentative;
+    private boolean representativeIsConst;
+
+    public EquivalenceClass(Collection<LogicalVariable> members, ILogicalExpression constRepresentative) {
+        this.members.addAll(members);
+        this.constRepresentative = constRepresentative;
+        representativeIsConst = true;
+    }
+
+    public EquivalenceClass(Collection<LogicalVariable> members, LogicalVariable variableRepresentative) {
+        this.members.addAll(members);
+        this.variableRepresentative = variableRepresentative;
+        representativeIsConst = false;
+    }
+
+    public EquivalenceClass(Collection<LogicalVariable> members, ILogicalExpression constRepresentative,
+            Collection<ILogicalExpression> expressionMembers) {
+        this(members, constRepresentative);
+        this.expressionMembers.addAll(expressionMembers);
+    }
+
+    public EquivalenceClass(Collection<LogicalVariable> members, LogicalVariable variableRepresentative,
+            Collection<ILogicalExpression> expressionMembers) {
+        this(members, variableRepresentative);
+        this.expressionMembers.addAll(expressionMembers);
+    }
+
+    public boolean representativeIsConst() {
+        return representativeIsConst;
+    }
+
+    public Collection<LogicalVariable> getMembers() {
+        return members;
+    }
+
+    public Collection<ILogicalExpression> getExpressionMembers() {
+        return expressionMembers;
+    }
+
+    public boolean contains(LogicalVariable var) {
+        return members.contains(var);
+    }
+
+    public boolean contains(ILogicalExpression expr) {
+        return expressionMembers.contains(expr);
+    }
+
+    public ILogicalExpression getConstRepresentative() {
+        return constRepresentative;
+    }
+
+    public LogicalVariable getVariableRepresentative() {
+        return variableRepresentative;
+    }
+
+    public void setConstRepresentative(ILogicalExpression constRepresentative) {
+        this.constRepresentative = constRepresentative;
+        this.representativeIsConst = true;
+    }
+
+    public void setVariableRepresentative(LogicalVariable variableRepresentative) {
+        this.variableRepresentative = variableRepresentative;
+        this.representativeIsConst = false;
+    }
+
+    public void merge(EquivalenceClass ec2) {
+        members.addAll(ec2.getMembers());
+        if (!representativeIsConst && ec2.representativeIsConst()) {
+            representativeIsConst = true;
+            constRepresentative = ec2.getConstRepresentative();
+        }
+        expressionMembers.addAll(ec2.getExpressionMembers());
+    }
+
+    public void addMember(LogicalVariable v) {
+        members.add(v);
+    }
+
+    public EquivalenceClass cloneEquivalenceClass() {
+        List<LogicalVariable> membersClone = new LinkedList<LogicalVariable>();
+        membersClone.addAll(members);
+        EquivalenceClass ec;
+        if (representativeIsConst()) {
+            ec = new EquivalenceClass(membersClone, constRepresentative);
+        } else {
+            ec = new EquivalenceClass(membersClone, variableRepresentative);
+        }
+        return ec;
+    }
+
+    @Override
+    public String toString() {
+        return "(<" + (representativeIsConst ? constRepresentative : variableRepresentative) + "> " + members + ";"
+                + expressionMembers + ")";
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof EquivalenceClass)) {
+            return false;
+        } else {
+            EquivalenceClass ec = (EquivalenceClass) obj;
+            if (!members.equals(ec.getMembers())) {
+                return false;
+            }
+            if (!expressionMembers.equals(ec.getExpressionMembers())) {
+                return false;
+            }
+            if (representativeIsConst) {
+                return ec.representativeIsConst() && (constRepresentative.equals(ec.getConstRepresentative()));
+            } else {
+                return !ec.representativeIsConst() && (variableRepresentative.equals(ec.getVariableRepresentative()));
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
new file mode 100644
index 0000000..4e341e4
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public interface IHyracksJobBuilder {
+    public enum TargetConstraint {
+        ONE,
+        SAME_COUNT
+    }
+
+    public void contributeHyracksOperator(ILogicalOperator op, IOperatorDescriptor opDesc);
+
+    public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apc);
+
+    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc);
+
+    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc,
+            AlgebricksPartitionConstraint pc);
+
+    /**
+     * inputs are numbered starting from 0
+     */
+    public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex);
+
+    public void contributeConnector(ILogicalOperator exchgOp, IConnectorDescriptor conn);
+
+    public void contributeConnectorWithTargetConstraint(ILogicalOperator exchgOp, IConnectorDescriptor conn,
+            TargetConstraint numberOfTargetPartitions);
+
+    public JobSpecification getJobSpec();
+
+    /**
+     * to be called only after all the graph information is added
+     */
+    public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalExpression.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
new file mode 100644
index 0000000..c4ae9bc
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+public interface ILogicalExpression {
+
+    public LogicalExpressionTag getExpressionTag();
+
+    public abstract <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException;
+
+    public void getUsedVariables(Collection<LogicalVariable> vars);
+
+    public void substituteVar(LogicalVariable v1, LogicalVariable v2);
+
+    // constraints (e.g., FDs, equivalences)
+
+    /**
+     * @param fds
+     *            Output argument: functional dependencies that can be inferred
+     *            from this expression.
+     * @param equivClasses
+     *            Output argument: Equivalence classes that can be inferred from
+     *            this expression.
+     */
+    public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
+            Map<LogicalVariable, EquivalenceClass> equivClasses);
+
+    /**
+     * @param fds
+     *            Output argument: functional dependencies that can be inferred
+     *            from this expression.
+     * @param outerVars
+     *            Input argument: variables coming from outer branch(es), e.g.,
+     *            the left branch of a left outer join.
+     */
+    public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars);
+
+    /**
+     * @param conjs
+     *            Output argument: a list of expression whose conjunction, in
+     *            any order, can replace the current expression.
+     * @return true if the expression can be broken in at least two conjuncts,
+     *         false otherwise.
+     */
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs);
+
+    public abstract ILogicalExpression cloneExpression();
+
+    public boolean isFunctional();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
new file mode 100644
index 0000000..d91cac0
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+
+public interface ILogicalOperator {
+
+    public LogicalOperatorTag getOperatorTag();
+
+    public ExecutionMode getExecutionMode();
+
+    public List<Mutable<ILogicalOperator>> getInputs();
+
+    boolean hasInputs();
+
+    public void recomputeSchema() throws AlgebricksException;
+
+    public List<LogicalVariable> getSchema();
+
+    /*
+     * 
+     * support for visitors
+     */
+
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException;
+
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException;
+
+    public boolean isMap();
+
+    public Map<String, Object> getAnnotations();
+
+    public void removeAnnotation(String annotationName);
+
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException;
+
+    // variables
+
+    /**
+     * Get the variable propogation policy from this operator's input to its
+     * output.
+     * 
+     * @return The VariablePropogationPolicy.
+     */
+    public VariablePropagationPolicy getVariablePropagationPolicy();
+
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException;
+
+    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException;
+
+    // structural properties
+
+    /**
+     * @return for each child, one vector of required physical properties
+     */
+
+    public PhysicalRequirements getRequiredPhysicalPropertiesForChildren(IPhysicalPropertiesVector requiredProperties);
+
+    /**
+     * @return the physical properties that this operator delivers, based on
+     *         what its children deliver
+     */
+
+    public IPhysicalPropertiesVector getDeliveredPhysicalProperties();
+
+    public void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException;
+
+    /**
+     * Indicates whether the expressions used by this operator must be variable reference expressions.
+     */
+    public boolean requiresVariableReferenceExpressions();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlan.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
new file mode 100644
index 0000000..6db8f8d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+public interface ILogicalPlan {
+    public List<Mutable<ILogicalOperator>> getRoots();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
new file mode 100644
index 0000000..06f619b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface ILogicalPlanAndMetadata {
+    public ILogicalPlan getPlan();
+
+    public IMetadataProvider<?, ?> getMetadataProvider();
+
+    public AlgebricksPartitionConstraint getClusterLocations();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
new file mode 100644
index 0000000..9b9273a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+
+public interface IOptimizationContext extends ITypingContext {
+
+    public abstract int getVarCounter();
+
+    public abstract void setVarCounter(int varCounter);
+
+    public abstract LogicalVariable newVar();
+
+    public abstract IMetadataProvider<?, ?> getMetadataProvider();
+
+    public abstract void setMetadataDeclarations(IMetadataProvider<?, ?> metadataProvider);
+
+    public abstract boolean checkIfInDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op);
+
+    public abstract void addToDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op);
+
+    /*
+     * returns true if op1 and op2 have already been compared
+     */
+    public abstract boolean checkAndAddToAlreadyCompared(ILogicalOperator op1, ILogicalOperator op2);
+    
+    public abstract void removeFromAlreadyCompared(ILogicalOperator op1);
+
+    public abstract void addNotToBeInlinedVar(LogicalVariable var);
+
+    public abstract boolean shouldNotBeInlined(LogicalVariable var);
+
+    public abstract void addPrimaryKey(FunctionalDependency pk);
+
+    public abstract List<LogicalVariable> findPrimaryKey(LogicalVariable recordVar);
+
+    public abstract void putEquivalenceClassMap(ILogicalOperator op, Map<LogicalVariable, EquivalenceClass> eqClassMap);
+
+    public abstract Map<LogicalVariable, EquivalenceClass> getEquivalenceClassMap(ILogicalOperator op);
+
+    public abstract void putFDList(ILogicalOperator op, List<FunctionalDependency> fdList);
+
+    public abstract List<FunctionalDependency> getFDList(ILogicalOperator op);
+
+    public abstract void putLogicalPropertiesVector(ILogicalOperator op, ILogicalPropertiesVector v);
+
+    public abstract ILogicalPropertiesVector getLogicalPropertiesVector(ILogicalOperator op);
+
+    public abstract IExpressionEvalSizeComputer getExpressionEvalSizeComputer();
+
+    public abstract IVariableEvalSizeEnvironment getVariableEvalSizeEnvironment();
+
+    public abstract IMergeAggregationExpressionFactory getMergeAggregationExpressionFactory();
+
+    public abstract PhysicalOptimizationConfig getPhysicalOptimizationConfig();
+
+    public abstract void invalidateTypeEnvironmentForOperator(ILogicalOperator op);
+
+    public abstract void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException;
+
+    public abstract void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars);
+    
+    public abstract LogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
new file mode 100644
index 0000000..dafe300
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+
+public interface IPhysicalOperator {
+
+    public PhysicalOperatorTag getOperatorTag();
+
+    /**
+     * @param op
+     *            the logical operator this physical operator annotates
+     * @param reqdByParent
+     *            parent's requirements, which are not enforced for now, as we
+     *            only explore one plan
+     * @return for each child, one vector of required physical properties
+     */
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent);
+
+    /**
+     * @return the physical properties that this operator delivers, based on
+     *         what its children deliver
+     */
+    public IPhysicalPropertiesVector getDeliveredProperties();
+
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException;
+
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException;
+
+    public void disableJobGenBelowMe();
+
+    public boolean isJobGenDisabledBelowMe();
+
+    public boolean isMicroOperator();
+
+    public void setHostQueryContext(Object context);
+
+    public Object getHostQueryContext();
+
+    /**
+     * @return labels (0 or 1) for each input and output indicating the dependency between them.
+     *         The edges labeled as 1 must wait for the edges with label 0.
+     */
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op);
+
+    /*
+     * This is needed to have a kind of cost based decision on whether to merge the shared subplans and materialize the result.
+     * If the subgraph whose result we would like to materialize has an operator that is computationally expensive, we assume
+     * it is cheaper to materialize the result of this subgraph and read from the file rather than recomputing it.
+     */
+    public boolean expensiveThanMaterialization();
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
new file mode 100644
index 0000000..83bc702
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public enum LogicalExpressionTag {
+    FUNCTION_CALL, VARIABLE, CONSTANT
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
new file mode 100644
index 0000000..9ddd927
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public enum LogicalOperatorTag {
+    AGGREGATE,
+    ASSIGN,
+    DATASOURCESCAN,
+    DISTINCT,
+    DISTRIBUTE_RESULT,
+    EMPTYTUPLESOURCE,
+    EXCHANGE,
+    EXTENSION_OPERATOR,
+    EXTERNAL_LOOKUP,
+    GROUP,
+    INDEX_INSERT_DELETE,
+    INNERJOIN,
+    INSERT_DELETE,
+    LEFTOUTERJOIN,
+    LIMIT,
+    MATERIALIZE,
+    NESTEDTUPLESOURCE,
+    ORDER,
+    PARTITIONINGSPLIT,
+    PROJECT,
+    REPLICATE,
+    RUNNINGAGGREGATE,
+    SCRIPT,
+    SELECT,
+    SINK,
+    SUBPLAN,
+    TOKENIZE,
+    UNIONALL,
+    UNNEST,
+    UNNEST_MAP,
+    UPDATE,
+    WRITE,
+    WRITE_RESULT,
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalVariable.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalVariable.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalVariable.java
new file mode 100644
index 0000000..07596b1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/LogicalVariable.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+/**
+ * Represents a logical variable in an asterix logical plan.
+ * 
+ * @author Vinayak R. Borkar [vborkar@ics.uci.edu]
+ */
+public final class LogicalVariable {
+    private final int id;
+
+    public LogicalVariable(int id) {
+        this.id = id;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    @Override
+    public String toString() {
+        return "$$" + id;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof LogicalVariable)) {
+            return false;
+        } else {
+            return id == ((LogicalVariable) obj).getId();
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        return id;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
new file mode 100644
index 0000000..251edca
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public interface OperatorAnnotations {
+    // hints
+    public static final String USE_HASH_GROUP_BY = "USE_HASH_GROUP_BY"; // -->
+    public static final String USE_EXTERNAL_GROUP_BY = "USE_EXTERNAL_GROUP_BY"; // -->
+    public static final String USE_RANGE_CONNECTOR = "USE_RANGE_CONNECTOR"; // -->
+    // Boolean
+    public static final String CARDINALITY = "CARDINALITY"; // -->
+    // Integer
+    public static final String MAX_NUMBER_FRAMES = "MAX_NUMBER_FRAMES"; // -->
+    // Integer
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
new file mode 100644
index 0000000..0c9e89a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public enum PhysicalOperatorTag {
+    AGGREGATE,
+    ASSIGN,
+    BROADCAST_EXCHANGE,
+    BTREE_SEARCH,
+    BULKLOAD,
+    DATASOURCE_SCAN,
+    DISTRIBUTE_RESULT,
+    EMPTY_TUPLE_SOURCE,
+    EXTENSION_OPERATOR,
+    EXTERNAL_GROUP_BY,
+    EXTERNAL_LOOKUP,
+    HASH_GROUP_BY,
+    HASH_PARTITION_EXCHANGE,
+    HASH_PARTITION_MERGE_EXCHANGE,
+    HDFS_READER,
+    HYBRID_HASH_JOIN,
+    IN_MEMORY_HASH_JOIN,
+    IN_MEMORY_STABLE_SORT,
+    INDEX_BULKLOAD,
+    INDEX_INSERT_DELETE,
+    INSERT_DELETE,
+    LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH,
+    MATERIALIZE,
+    MICRO_PRE_CLUSTERED_GROUP_BY,
+    NESTED_LOOP,
+    NESTED_TUPLE_SOURCE,
+    ONE_TO_ONE_EXCHANGE,
+    PARTITIONINGSPLIT,
+    PRE_CLUSTERED_GROUP_BY,
+    PRE_SORTED_DISTINCT_BY,
+    RANDOM_PARTITION_EXCHANGE,
+    RANDOM_MERGE_EXCHANGE,
+    RANGE_PARTITION_EXCHANGE,
+    RANGE_PARTITION_MERGE_EXCHANGE,
+    RTREE_SEARCH,
+    RUNNING_AGGREGATE,
+    SINGLE_PARTITION_INVERTED_INDEX_SEARCH,
+    SINK,
+    SINK_WRITE,
+    SORT_GROUP_BY,
+    SORT_MERGE_EXCHANGE,
+    SPLIT,
+    STABLE_SORT,
+    STATS,
+    STREAM_LIMIT,
+    STREAM_PROJECT,
+    STREAM_SELECT,
+    STRING_STREAM_SCRIPT,
+    SUBPLAN,
+    TOKENIZE,
+    UNION_ALL,
+    UNNEST,
+    UPDATE,
+    WRITE_RESULT,
+}

[40/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
deleted file mode 100644
index 993e8d7..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-@SuppressWarnings("rawtypes")
-public class InsertDeletePOperator extends AbstractPhysicalOperator {
-
-    private LogicalVariable payload;
-    private List<LogicalVariable> keys;
-    private IDataSource<?> dataSource;
-    private final List<LogicalVariable> additionalFilteringKeys;
-
-    public InsertDeletePOperator(LogicalVariable payload, List<LogicalVariable> keys,
-            List<LogicalVariable> additionalFilteringKeys, IDataSource dataSource) {
-        this.payload = payload;
-        this.keys = keys;
-        this.dataSource = dataSource;
-        this.additionalFilteringKeys = additionalFilteringKeys;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.INSERT_DELETE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
-        scanVariables.addAll(keys);
-        scanVariables.add(new LogicalVariable(-1));
-        IPhysicalPropertiesVector r = dataSource.getPropertiesProvider().computePropertiesVector(scanVariables);
-        r.getLocalProperties().clear();
-        IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
-        requirements[0] = r;
-        return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @SuppressWarnings("unchecked")
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
-        IMetadataProvider mp = context.getMetadataProvider();
-        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
-        JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
-
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
-        if (insertDeleteOp.getOperation() == Kind.INSERT) {
-            runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
-                    additionalFilteringKeys, inputDesc, context, spec, false);
-        } else {
-            runtimeAndConstraints = mp.getDeleteRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
-                    additionalFilteringKeys, inputDesc, context, spec);
-        }
-
-        builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
-        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java
deleted file mode 100644
index 27f4b25..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MaterializePOperator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.MaterializingOperatorDescriptor;
-
-public class MaterializePOperator extends AbstractPhysicalOperator {
-
-    private final boolean isSingleActivity;
-
-    public MaterializePOperator(boolean isSingleActivity) {
-        this.isSingleActivity = isSingleActivity;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.MATERIALIZE;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
-                propagatedSchema, context);
-        MaterializingOperatorDescriptor materializationOpDesc = new MaterializingOperatorDescriptor(
-                builder.getJobSpec(), recDescriptor, isSingleActivity);
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, materializationOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[] { 0 };
-        int[] outputDependencyLabels;
-        if (isSingleActivity) {
-            outputDependencyLabels = new int[] { 0 };
-        } else {
-            outputDependencyLabels = new int[] { 1 };
-        }
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
deleted file mode 100644
index 546a77b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-
-public class MicroPreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
-
-    public MicroPreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
-        super(columnList);
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
-        GroupByOperator gby = (GroupByOperator) op;
-        int numFds = gby.getDecorList().size();
-        int fdColumns[] = new int[numFds];
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        int j = 0;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new AlgebricksException("pre-sorted group-by expects variable references.");
-            }
-            VariableReferenceExpression v = (VariableReferenceExpression) expr;
-            LogicalVariable decor = v.getVariableReference();
-            fdColumns[j++] = inputSchemas[0].findVariable(decor);
-        }
-        // compile subplans and set the gby op. schema accordingly
-        AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
-        IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys,
-                fdColumns);
-
-        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
-                columnList, env, context);
-        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()),
-                inputSchemas[0], context);
-        MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys,
-                comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null);
-        builder.contributeMicroOperator(gby, runtime, recordDescriptor);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
deleted file mode 100644
index a2e697d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspector;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.primitive.VoidPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
-
-/**
- * Left input is broadcast and preserves its local properties.
- * Right input can be partitioned in any way.
- */
-public class NLJoinPOperator extends AbstractJoinPOperator {
-
-    private final int memSize;
-
-    public NLJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType, int memSize) {
-        super(kind, partitioningType);
-        this.memSize = memSize;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.NESTED_LOOP;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context) {
-        if (partitioningType != JoinPartitioningType.BROADCAST) {
-            throw new NotImplementedException(partitioningType + " nested loop joins are not implemented.");
-        }
-
-        IPartitioningProperty pp;
-
-        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
-
-        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
-            IPhysicalPropertiesVector pv1 = op2.getPhysicalOperator().getDeliveredProperties();
-            if (pv1 == null) {
-                pp = null;
-            } else {
-                pp = pv1.getPartitioningProperty();
-            }
-        } else {
-            pp = IPartitioningProperty.UNPARTITIONED;
-        }
-
-        List<ILocalStructuralProperty> localProps = new LinkedList<ILocalStructuralProperty>();
-        this.deliveredProperties = new StructuralPropertiesVector(pp, localProps);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        if (partitioningType != JoinPartitioningType.BROADCAST) {
-            throw new NotImplementedException(partitioningType + " nested loop joins are not implemented.");
-        }
-
-        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
-        pv[0] = new StructuralPropertiesVector(new BroadcastPartitioningProperty(null), null);
-        pv[1] = new StructuralPropertiesVector(null, null);
-        return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
-                propagatedSchema, context);
-        IOperatorSchema[] conditionInputSchemas = new IOperatorSchema[1];
-        conditionInputSchemas[0] = propagatedSchema;
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(join.getCondition().getValue(),
-                context.getTypeEnvironment(op), conditionInputSchemas, context);
-        ITuplePairComparatorFactory comparatorFactory = new TuplePairEvaluatorFactory(cond,
-                context.getBinaryBooleanInspectorFactory());
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        IOperatorDescriptor opDesc = null;
-
-        switch (kind) {
-            case INNER: {
-                opDesc = new NestedLoopJoinOperatorDescriptor(spec, comparatorFactory, recDescriptor, memSize, false,
-                        null);
-                break;
-            }
-            case LEFT_OUTER: {
-                INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
-                for (int j = 0; j < nullWriterFactories.length; j++) {
-                    nullWriterFactories[j] = context.getNullWriterFactory();
-                }
-                opDesc = new NestedLoopJoinOperatorDescriptor(spec, comparatorFactory, recDescriptor, memSize, true,
-                        nullWriterFactories);
-                break;
-            }
-            default: {
-                throw new NotImplementedException();
-            }
-        }
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-
-        ILogicalOperator src1 = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getValue();
-        builder.contributeGraphEdge(src2, 0, op, 1);
-    }
-
-    public static class TuplePairEvaluatorFactory implements ITuplePairComparatorFactory {
-
-        private static final long serialVersionUID = 1L;
-        private final IScalarEvaluatorFactory cond;
-        private final IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory;
-
-        public TuplePairEvaluatorFactory(IScalarEvaluatorFactory cond,
-                IBinaryBooleanInspectorFactory binaryBooleanInspectorFactory) {
-            this.cond = cond;
-            this.binaryBooleanInspectorFactory = binaryBooleanInspectorFactory;
-        }
-
-        @Override
-        public synchronized ITuplePairComparator createTuplePairComparator(IHyracksTaskContext ctx) {
-            return new TuplePairEvaluator(ctx, cond, binaryBooleanInspectorFactory.createBinaryBooleanInspector(ctx));
-        }
-    }
-
-    public static class TuplePairEvaluator implements ITuplePairComparator {
-        private final IHyracksTaskContext ctx;
-        private IScalarEvaluator condEvaluator;
-        private final IScalarEvaluatorFactory condFactory;
-        private final IPointable p;
-        private final CompositeFrameTupleReference compositeTupleRef;
-        private final FrameTupleReference leftRef;
-        private final FrameTupleReference rightRef;
-        private final IBinaryBooleanInspector binaryBooleanInspector;
-
-        public TuplePairEvaluator(IHyracksTaskContext ctx, IScalarEvaluatorFactory condFactory,
-                IBinaryBooleanInspector binaryBooleanInspector) {
-            this.ctx = ctx;
-            this.condFactory = condFactory;
-            this.binaryBooleanInspector = binaryBooleanInspector;
-            this.leftRef = new FrameTupleReference();
-            this.p = VoidPointable.FACTORY.createPointable();
-            this.rightRef = new FrameTupleReference();
-            this.compositeTupleRef = new CompositeFrameTupleReference(leftRef, rightRef);
-        }
-
-        @Override
-        public int compare(IFrameTupleAccessor outerAccessor, int outerIndex, IFrameTupleAccessor innerAccessor,
-                int innerIndex) throws HyracksDataException {
-            if (condEvaluator == null) {
-                try {
-                    this.condEvaluator = condFactory.createScalarEvaluator(ctx);
-                } catch (AlgebricksException ae) {
-                    throw new HyracksDataException(ae);
-                }
-            }
-            compositeTupleRef.reset(outerAccessor, outerIndex, innerAccessor, innerIndex);
-            try {
-                condEvaluator.evaluate(compositeTupleRef, p);
-            } catch (AlgebricksException ae) {
-                throw new HyracksDataException(ae);
-            }
-            boolean result = binaryBooleanInspector
-                    .getBooleanValue(p.getByteArray(), p.getStartOffset(), p.getLength());
-            if (result)
-                return 0;
-            else
-                return 1;
-        }
-    }
-
-    public static class CompositeFrameTupleReference implements IFrameTupleReference {
-
-        private final FrameTupleReference refLeft;
-        private final FrameTupleReference refRight;
-
-        public CompositeFrameTupleReference(FrameTupleReference refLeft, FrameTupleReference refRight) {
-            this.refLeft = refLeft;
-            this.refRight = refRight;
-        }
-
-        public void reset(IFrameTupleAccessor outerAccessor, int outerIndex, IFrameTupleAccessor innerAccessor,
-                int innerIndex) {
-            refLeft.reset(outerAccessor, outerIndex);
-            refRight.reset(innerAccessor, innerIndex);
-        }
-
-        @Override
-        public int getFieldCount() {
-            return refLeft.getFieldCount() + refRight.getFieldCount();
-        }
-
-        @Override
-        public byte[] getFieldData(int fIdx) {
-            int leftFieldCount = refLeft.getFieldCount();
-            if (fIdx < leftFieldCount)
-                return refLeft.getFieldData(fIdx);
-            else
-                return refRight.getFieldData(fIdx - leftFieldCount);
-        }
-
-        @Override
-        public int getFieldStart(int fIdx) {
-            int leftFieldCount = refLeft.getFieldCount();
-            if (fIdx < leftFieldCount)
-                return refLeft.getFieldStart(fIdx);
-            else
-                return refRight.getFieldStart(fIdx - leftFieldCount);
-        }
-
-        @Override
-        public int getFieldLength(int fIdx) {
-            int leftFieldCount = refLeft.getFieldCount();
-            if (fIdx < leftFieldCount)
-                return refLeft.getFieldLength(fIdx);
-            else
-                return refRight.getFieldLength(fIdx - leftFieldCount);
-        }
-
-        @Override
-        public IFrameTupleAccessor getFrameTupleAccessor() {
-            throw new NotImplementedException();
-        }
-
-        @Override
-        public int getTupleIndex() {
-            throw new NotImplementedException();
-        }
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
deleted file mode 100644
index 4fdfa32..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.NestedTupleSourceRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class NestedTupleSourcePOperator extends AbstractPhysicalOperator {
-
-    public NestedTupleSourcePOperator() {
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.NESTED_TUPLE_SOURCE;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        Mutable<ILogicalOperator> dataSource = ((NestedTupleSourceOperator) op).getDataSourceReference();
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) dataSource.getValue().getInputs().get(0).getValue();
-        IPhysicalPropertiesVector inheritedProps = op2.getDeliveredPhysicalProperties();
-        AbstractLogicalOperator parent = (AbstractLogicalOperator) dataSource.getValue();
-        if (parent.getOperatorTag() == LogicalOperatorTag.GROUP) {
-            // The following part computes the data property regarding to each particular group.
-            // TODO(buyingyi): we need to add the original data property as well. But currently
-            // there are places assuming there is only one LocalOrderProperty and one
-            // LocalGroupingProperty delivered by an operator.
-            GroupByOperator gby = (GroupByOperator) parent;
-            List<ILocalStructuralProperty> originalLocalProperties = inheritedProps.getLocalProperties();
-            List<ILocalStructuralProperty> newLocalProperties = null;
-            if (originalLocalProperties != null) {
-                newLocalProperties = new ArrayList<ILocalStructuralProperty>();
-                for (ILocalStructuralProperty lsp : inheritedProps.getLocalProperties()) {
-                    ILocalStructuralProperty newLsp = lsp.regardToGroup(gby.getGbyVarList());
-                    if (newLsp != null) {
-                        newLocalProperties.add(newLsp);
-                    }
-                }
-            }
-            deliveredProperties = new StructuralPropertiesVector(inheritedProps.getPartitioningProperty(),
-                    newLocalProperties);
-        } else {
-            deliveredProperties = inheritedProps.clone();
-        }
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return null;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        propagatedSchema.addAllVariables(outerPlanSchema);
-        NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory();
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
-                context);
-        builder.contributeMicroOperator(op, runtime, recDesc);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
deleted file mode 100644
index 456744e..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
-
-public class OneToOneExchangePOperator extends AbstractExchangePOperator {
-
-    public OneToOneExchangePOperator() {
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
-        IConnectorDescriptor conn = new OneToOneConnectorDescriptor(spec);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.SAME_COUNT);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
deleted file mode 100644
index a24c48d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
-
-public class PreSortedDistinctByPOperator extends AbstractPhysicalOperator {
-
-    private List<LogicalVariable> columnList;
-
-    public PreSortedDistinctByPOperator(List<LogicalVariable> columnList) {
-        this.columnList = columnList;
-    }
-
-    public void setDistinctByColumns(List<LogicalVariable> distinctByColumns) {
-        this.columnList = distinctByColumns;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
-        List<ILocalStructuralProperty> propsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
-        deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
-        List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>();
-        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-        for (LogicalVariable column : columnList) {
-            orderColumns.add(new OrderColumn(column, OrderKind.ASC));
-        }
-        localProps.add(new LocalOrderProperty(orderColumns));
-        IPartitioningProperty pp = null;
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
-        if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
-            pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), null);
-        }
-        pv[0] = new StructuralPropertiesVector(pp, localProps);
-        return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
-        int sz = inputSchemas[0].getSize();
-        int fdSz = sz - columnList.size();
-        int[] fdColumns = new int[fdSz];
-        int j = 0;
-        for (LogicalVariable v : inputSchemas[0]) {
-            if (!columnList.contains(v)) {
-                fdColumns[j++] = inputSchemas[0].findVariable(v);
-            }
-        }
-        int[] keysAndDecs = new int[keys.length + fdColumns.length];
-        for (int i = 0; i < keys.length; i++) {
-            keysAndDecs[i] = keys[i];
-        }
-        for (int i = 0; i < fdColumns.length; i++) {
-            keysAndDecs[i + keys.length] = fdColumns[i];
-        }
-
-        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
-                columnList, context.getTypeEnvironment(op), context);
-        IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {};
-        IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(
-                aggFactories, keysAndDecs);
-
-        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema,
-                context);
-        /** make fd columns part of the key but the comparator only compares the distinct key columns */
-        PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keysAndDecs,
-                comparatorFactories, aggregatorFactory, recordDescriptor);
-
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
deleted file mode 100644
index bcd998d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansRunningAggregatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
-
-public class PreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
-
-    public PreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
-        super(columnList);
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
-        GroupByOperator gby = (GroupByOperator) op;
-        int numFds = gby.getDecorList().size();
-        int fdColumns[] = new int[numFds];
-        int j = 0;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new AlgebricksException("pre-sorted group-by expects variable references.");
-            }
-            VariableReferenceExpression v = (VariableReferenceExpression) expr;
-            LogicalVariable decor = v.getVariableReference();
-            fdColumns[j++] = inputSchemas[0].findVariable(decor);
-        }
-        // compile subplans and set the gby op. schema accordingly
-        AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
-        IAggregatorDescriptorFactory aggregatorFactory;
-
-        if (((AbstractLogicalOperator) (gby.getNestedPlans().get(0).getRoots().get(0).getValue())).getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) {
-            aggregatorFactory = new NestedPlansRunningAggregatorFactory(subplans, keys, fdColumns);
-        } else {
-            aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns);
-        }
-
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
-                columnList, context.getTypeEnvironment(op), context);
-        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema,
-                context);
-
-        PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys,
-                comparatorFactories, aggregatorFactory, recordDescriptor);
-
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
deleted file mode 100644
index 2478859..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
-
-public class RandomMergeExchangePOperator extends AbstractExchangePOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.RANDOM_MERGE_EXCHANGE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        this.deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
-                new ArrayList<ILocalStructuralProperty>(0));
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
-        IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java
deleted file mode 100644
index c633f7b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.RandomPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
-
-public class RandomPartitionPOperator extends AbstractExchangePOperator {
-
-    private final INodeDomain domain;
-
-    public RandomPartitionPOperator(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op,
-                opSchema, context);
-        builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        ITuplePartitionComputerFactory tpcf = new RandomPartitionComputerFactory(domain.cardinality());
-        MToNPartitioningConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.RANDOM_PARTITION_EXCHANGE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        this.deliveredProperties = new StructuralPropertiesVector(new RandomPartitioningProperty(domain), op2
-                .getDeliveredPhysicalProperties().getLocalProperties());
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java
deleted file mode 100644
index 99f68e9..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.range.FieldRangePartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.range.IRangeMap;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
-
-public class RangePartitionMergePOperator extends AbstractExchangePOperator {
-
-    private List<OrderColumn> partitioningFields;
-    private INodeDomain domain;
-    private IRangeMap rangeMap;
-
-    public RangePartitionMergePOperator(List<OrderColumn> partitioningFields, INodeDomain domain, IRangeMap rangeMap) {
-        this.partitioningFields = partitioningFields;
-        this.domain = domain;
-        this.rangeMap = rangeMap;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.RANGE_PARTITION_MERGE_EXCHANGE;
-    }
-
-    public List<OrderColumn> getPartitioningFields() {
-        return partitioningFields;
-    }
-
-    public INodeDomain getDomain() {
-        return domain;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        List<LogicalVariable> varList = new ArrayList<LogicalVariable>();
-        for (OrderColumn oc : partitioningFields) {
-            varList.add(oc.getColumn());
-        }
-        IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(varList), domain);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
-        List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
-        for (ILocalStructuralProperty prop : op2Locals) {
-            if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
-                locals.add(prop);
-            } else {
-                break;
-            }
-        }
-
-        this.deliveredProperties = new StructuralPropertiesVector(p, locals);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<ILocalStructuralProperty> orderProps = new LinkedList<ILocalStructuralProperty>();
-        List<OrderColumn> columns = new ArrayList<OrderColumn>();
-        for (OrderColumn oc : partitioningFields) {
-            LogicalVariable var = oc.getColumn();
-            columns.add(new OrderColumn(var, oc.getOrder()));
-        }
-        orderProps.add(new LocalOrderProperty(columns));
-        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
-                orderProps) };
-        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        int n = partitioningFields.size();
-        int[] sortFields = new int[n];
-        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
-
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        INormalizedKeyComputerFactory nkcf = null;
-
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        int i = 0;
-        for (OrderColumn oc : partitioningFields) {
-            LogicalVariable var = oc.getColumn();
-            sortFields[i] = opSchema.findVariable(var);
-            Object type = env.getVarType(var);
-            OrderKind order = oc.getOrder();
-            if (i == 0 && nkcfProvider != null && type != null) {
-                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
-            }
-            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
-            i++;
-        }
-        ITuplePartitionComputerFactory tpcf = new FieldRangePartitionComputerFactory(sortFields, comps, rangeMap);
-        IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " " + partitioningFields + " SPLIT COUNT:" + rangeMap.getSplitCount();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
deleted file mode 100644
index 66955e9..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.range.FieldRangePartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.range.IRangeMap;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
-
-public class RangePartitionPOperator extends AbstractExchangePOperator {
-
-    private List<OrderColumn> partitioningFields;
-    private INodeDomain domain;
-    private IRangeMap rangeMap;
-
-    public RangePartitionPOperator(List<OrderColumn> partitioningFields, INodeDomain domain, IRangeMap rangeMap) {
-        this.partitioningFields = partitioningFields;
-        this.domain = domain;
-        this.rangeMap = rangeMap;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.RANGE_PARTITION_EXCHANGE;
-    }
-
-    public List<OrderColumn> getPartitioningFields() {
-        return partitioningFields;
-    }
-
-    public INodeDomain getDomain() {
-        return domain;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        IPartitioningProperty p = new OrderedPartitionedProperty(new ArrayList<OrderColumn>(partitioningFields), domain);
-        this.deliveredProperties = new StructuralPropertiesVector(p, new LinkedList<ILocalStructuralProperty>());
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        int n = partitioningFields.size();
-        int[] sortFields = new int[n];
-        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
-
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        INormalizedKeyComputerFactory nkcf = null;
-
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        int i = 0;
-        for (OrderColumn oc : partitioningFields) {
-            LogicalVariable var = oc.getColumn();
-            sortFields[i] = opSchema.findVariable(var);
-            Object type = env.getVarType(var);
-            OrderKind order = oc.getOrder();
-            if (i == 0 && nkcfProvider != null && type != null) {
-                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
-            }
-            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
-            i++;
-        }
-        ITuplePartitionComputerFactory tpcf = new FieldRangePartitionComputerFactory(sortFields, comps, rangeMap);
-        IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " " + partitioningFields + " SPLIT COUNT:" + rangeMap.getSplitCount();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
deleted file mode 100644
index d8f7d99..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.misc.SplitOperatorDescriptor;
-
-public class ReplicatePOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.SPLIT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
-
-        ReplicateOperator rop = (ReplicateOperator) op;
-        int outputArity = rop.getOutputArity();
-        boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags();
-
-        SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity, outputMaterializationFlags);
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, splitOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[] { 0 };
-        ReplicateOperator rop = (ReplicateOperator) op;
-        int[] outputDependencyLabels = new int[rop.getOutputArity()];
-        // change the labels of outputs that requires materialization to 1
-        boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags();
-        for (int i = 0; i < rop.getOutputArity(); i++) {
-            if (outputMaterializationFlags[i]) {
-                outputDependencyLabels[i] = 1;
-            }
-        }
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

[21/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
new file mode 100644
index 0000000..4cb111d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
+import edu.uci.ics.hyracks.api.dataflow.connectors.PipeliningConnectorPolicy;
+import edu.uci.ics.hyracks.api.dataflow.connectors.SendSideMaterializedPipeliningConnectorPolicy;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class ConnectorPolicyAssignmentPolicy implements IConnectorPolicyAssignmentPolicy {
+    private static final long serialVersionUID = 1L;
+    private IConnectorPolicy senderSideMaterializePolicy = new SendSideMaterializedPipeliningConnectorPolicy();
+    private IConnectorPolicy pipeliningPolicy = new PipeliningConnectorPolicy();
+
+    @Override
+    public IConnectorPolicy getConnectorPolicyAssignment(IConnectorDescriptor c, int nProducers, int nConsumers,
+            int[] fanouts) {
+        if (c instanceof MToNPartitioningMergingConnectorDescriptor) {
+            return senderSideMaterializePolicy;
+        } else {
+            return pipeliningPolicy;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
new file mode 100644
index 0000000..e1f897c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
@@ -0,0 +1,330 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class JobBuilder implements IHyracksJobBuilder {
+
+    private JobSpecification jobSpec;
+    private AlgebricksPartitionConstraint clusterLocations;
+
+    private Map<ILogicalOperator, ArrayList<ILogicalOperator>> outEdges = new HashMap<ILogicalOperator, ArrayList<ILogicalOperator>>();
+    private Map<ILogicalOperator, ArrayList<ILogicalOperator>> inEdges = new HashMap<ILogicalOperator, ArrayList<ILogicalOperator>>();
+    private Map<ILogicalOperator, Pair<IConnectorDescriptor, TargetConstraint>> connectors = new HashMap<ILogicalOperator, Pair<IConnectorDescriptor, TargetConstraint>>();
+
+    private Map<ILogicalOperator, Pair<IPushRuntimeFactory, RecordDescriptor>> microOps = new HashMap<ILogicalOperator, Pair<IPushRuntimeFactory, RecordDescriptor>>();
+    private Map<IPushRuntimeFactory, ILogicalOperator> revMicroOpMap = new HashMap<IPushRuntimeFactory, ILogicalOperator>();
+    private Map<ILogicalOperator, IOperatorDescriptor> hyracksOps = new HashMap<ILogicalOperator, IOperatorDescriptor>();
+    private Map<ILogicalOperator, AlgebricksPartitionConstraint> pcForMicroOps = new HashMap<ILogicalOperator, AlgebricksPartitionConstraint>();
+
+    private int aodCounter = 0;
+    private Map<ILogicalOperator, Integer> algebraicOpBelongingToMetaAsterixOp = new HashMap<ILogicalOperator, Integer>();
+    private Map<Integer, List<Pair<IPushRuntimeFactory, RecordDescriptor>>> metaAsterixOpSkeletons = new HashMap<Integer, List<Pair<IPushRuntimeFactory, RecordDescriptor>>>();
+    private Map<Integer, AlgebricksMetaOperatorDescriptor> metaAsterixOps = new HashMap<Integer, AlgebricksMetaOperatorDescriptor>();
+    private final Map<IOperatorDescriptor, AlgebricksPartitionConstraint> partitionConstraintMap = new HashMap<IOperatorDescriptor, AlgebricksPartitionConstraint>();
+
+    public JobBuilder(JobSpecification jobSpec, AlgebricksPartitionConstraint clusterLocations) {
+        this.jobSpec = jobSpec;
+        this.clusterLocations = clusterLocations;
+    }
+
+    @Override
+    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc) {
+        contributeMicroOperator(op, runtime, recDesc, null);
+    }
+
+    @Override
+    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc,
+            AlgebricksPartitionConstraint pc) {
+        microOps.put(op, new Pair<IPushRuntimeFactory, RecordDescriptor>(runtime, recDesc));
+        revMicroOpMap.put(runtime, op);
+        if (pc != null) {
+            pcForMicroOps.put(op, pc);
+        }
+        AbstractLogicalOperator logicalOp = (AbstractLogicalOperator) op;
+        if (logicalOp.getExecutionMode() == ExecutionMode.UNPARTITIONED && pc == null) {
+            AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1);
+            pcForMicroOps.put(logicalOp, apc);
+        }
+    }
+
+    @Override
+    public void contributeConnector(ILogicalOperator exchgOp, IConnectorDescriptor conn) {
+        connectors.put(exchgOp, new Pair<IConnectorDescriptor, TargetConstraint>(conn, null));
+    }
+
+    @Override
+    public void contributeConnectorWithTargetConstraint(ILogicalOperator exchgOp, IConnectorDescriptor conn,
+            TargetConstraint numberOfTargetPartitions) {
+        connectors.put(exchgOp, new Pair<IConnectorDescriptor, TargetConstraint>(conn, numberOfTargetPartitions));
+    }
+
+    @Override
+    public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex) {
+        ArrayList<ILogicalOperator> outputs = outEdges.get(src);
+        if (outputs == null) {
+            outputs = new ArrayList<ILogicalOperator>();
+            outEdges.put(src, outputs);
+        }
+        addAtPos(outputs, dest, srcOutputIndex);
+
+        ArrayList<ILogicalOperator> inp = inEdges.get(dest);
+        if (inp == null) {
+            inp = new ArrayList<ILogicalOperator>();
+            inEdges.put(dest, inp);
+        }
+        addAtPos(inp, src, destInputIndex);
+    }
+
+    @Override
+    public void contributeHyracksOperator(ILogicalOperator op, IOperatorDescriptor opDesc) {
+        hyracksOps.put(op, opDesc);
+    }
+
+    @Override
+    public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apc) {
+        partitionConstraintMap.put(opDesc, apc);
+    }
+
+    @Override
+    public JobSpecification getJobSpec() {
+        return jobSpec;
+    }
+
+    @Override
+    public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException {
+        buildAsterixComponents();
+        Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = setupConnectors();
+        for (ILogicalOperator r : roots) {
+            IOperatorDescriptor opDesc = findOpDescForAlgebraicOp(r);
+            jobSpec.addRoot(opDesc);
+        }
+        setAllPartitionConstraints(tgtConstraints);
+    }
+
+    private void setAllPartitionConstraints(Map<IConnectorDescriptor, TargetConstraint> tgtConstraints) {
+        List<OperatorDescriptorId> roots = jobSpec.getRoots();
+        setSpecifiedPartitionConstraints();
+        for (OperatorDescriptorId rootId : roots) {
+            setPartitionConstraintsDFS(rootId, tgtConstraints, null);
+        }
+    }
+
+    private void setSpecifiedPartitionConstraints() {
+        for (ILogicalOperator op : pcForMicroOps.keySet()) {
+            AlgebricksPartitionConstraint pc = pcForMicroOps.get(op);
+            Integer k = algebraicOpBelongingToMetaAsterixOp.get(op);
+            AlgebricksMetaOperatorDescriptor amod = metaAsterixOps.get(k);
+            partitionConstraintMap.put(amod, pc);
+        }
+        for (IOperatorDescriptor opDesc : partitionConstraintMap.keySet()) {
+            AlgebricksPartitionConstraint pc = partitionConstraintMap.get(opDesc);
+            AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, pc);
+        }
+    }
+
+    private void setPartitionConstraintsDFS(OperatorDescriptorId opId,
+            Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp) {
+        List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
+        AlgebricksPartitionConstraint opConstraint = null;
+        IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
+        if (opInputs != null) {
+            for (IConnectorDescriptor conn : opInputs) {
+                ConnectorDescriptorId cid = conn.getConnectorId();
+                org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec
+                        .getConnectorOperatorMap().get(cid);
+                IOperatorDescriptor src = p.getLeft().getLeft();
+                // DFS
+                setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc);
+
+                TargetConstraint constraint = tgtConstraints.get(conn);
+                if (constraint != null) {
+                    switch (constraint) {
+                        case ONE: {
+                            opConstraint = new AlgebricksCountPartitionConstraint(1);
+                            break;
+                        }
+                        case SAME_COUNT: {
+                            opConstraint = partitionConstraintMap.get(src);
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        if (partitionConstraintMap.get(opDesc) == null) {
+            if (opConstraint == null) {
+                if (parentOp != null) {
+                    AlgebricksPartitionConstraint pc = partitionConstraintMap.get(parentOp);
+                    if (pc != null) {
+                        opConstraint = pc;
+                    } else if (opInputs == null || opInputs.size() == 0) {
+                        opConstraint = new AlgebricksCountPartitionConstraint(1);
+                    }
+                }
+                if (opConstraint == null) {
+                    opConstraint = clusterLocations;
+                }
+            }
+            partitionConstraintMap.put(opDesc, opConstraint);
+            AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, opConstraint);
+        }
+    }
+
+    private Map<IConnectorDescriptor, TargetConstraint> setupConnectors() throws AlgebricksException {
+        Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = new HashMap<IConnectorDescriptor, TargetConstraint>();
+        for (ILogicalOperator exchg : connectors.keySet()) {
+            ILogicalOperator inOp = inEdges.get(exchg).get(0);
+            ILogicalOperator outOp = outEdges.get(exchg).get(0);
+            IOperatorDescriptor inOpDesc = findOpDescForAlgebraicOp(inOp);
+            IOperatorDescriptor outOpDesc = findOpDescForAlgebraicOp(outOp);
+            Pair<IConnectorDescriptor, TargetConstraint> connPair = connectors.get(exchg);
+            IConnectorDescriptor conn = connPair.first;
+            int producerPort = outEdges.get(inOp).indexOf(exchg);
+            int consumerPort = inEdges.get(outOp).indexOf(exchg);
+            jobSpec.connect(conn, inOpDesc, producerPort, outOpDesc, consumerPort);
+            if (connPair.second != null) {
+                tgtConstraints.put(conn, connPair.second);
+            }
+        }
+        return tgtConstraints;
+    }
+
+    private IOperatorDescriptor findOpDescForAlgebraicOp(ILogicalOperator op) throws AlgebricksException {
+        IOperatorDescriptor hOpDesc = hyracksOps.get(op);
+        if (hOpDesc != null) {
+            return hOpDesc;
+        }
+        Integer metaOpKey = algebraicOpBelongingToMetaAsterixOp.get(op);
+        if (metaOpKey == null) {
+            throw new AlgebricksException("Could not generate operator descriptor for operator " + op);
+        }
+        return metaAsterixOps.get(metaOpKey);
+    }
+
+    private void buildAsterixComponents() {
+        for (ILogicalOperator aop : microOps.keySet()) {
+            addMicroOpToMetaRuntimeOp(aop);
+        }
+        for (Integer k : metaAsterixOpSkeletons.keySet()) {
+            List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents = metaAsterixOpSkeletons.get(k);
+            AlgebricksMetaOperatorDescriptor amod = buildMetaAsterixOpDesc(opContents);
+            metaAsterixOps.put(k, amod);
+        }
+    }
+
+    private AlgebricksMetaOperatorDescriptor buildMetaAsterixOpDesc(
+            List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents) {
+        // RecordDescriptor outputRecordDesc = null;
+        int n = opContents.size();
+        IPushRuntimeFactory[] runtimeFactories = new IPushRuntimeFactory[n];
+        RecordDescriptor[] internalRecordDescriptors = new RecordDescriptor[n];
+        int i = 0;
+        for (Pair<IPushRuntimeFactory, RecordDescriptor> p : opContents) {
+            runtimeFactories[i] = p.first;
+            internalRecordDescriptors[i] = p.second;
+            // if (i == n - 1) {
+            // outputRecordDesc = p.second;
+            // }
+            i++;
+        }
+        ILogicalOperator lastLogicalOp = revMicroOpMap.get(runtimeFactories[n - 1]);
+        ArrayList<ILogicalOperator> outOps = outEdges.get(lastLogicalOp);
+        int outArity = (outOps == null) ? 0 : outOps.size();
+        ILogicalOperator firstLogicalOp = revMicroOpMap.get(runtimeFactories[0]);
+        ArrayList<ILogicalOperator> inOps = inEdges.get(firstLogicalOp);
+        int inArity = (inOps == null) ? 0 : inOps.size();
+        // boolean isLeafOp = inEdges.get(firstLogicalOp) == null;
+        return new AlgebricksMetaOperatorDescriptor(jobSpec, inArity, outArity, runtimeFactories,
+                internalRecordDescriptors);
+    }
+
+    private void addMicroOpToMetaRuntimeOp(ILogicalOperator aop) {
+        Integer k = algebraicOpBelongingToMetaAsterixOp.get(aop);
+        if (k == null) {
+            k = createNewMetaOpInfo(aop);
+        }
+        ArrayList<ILogicalOperator> destList = outEdges.get(aop);
+        if (destList == null || destList.size() != 1) {
+            // for now, we only support linear plans inside meta-ops.
+            return;
+        }
+        ILogicalOperator dest = destList.get(0);
+        Integer j = algebraicOpBelongingToMetaAsterixOp.get(dest);
+        if (j == null && microOps.get(dest) != null) {
+            algebraicOpBelongingToMetaAsterixOp.put(dest, k);
+            List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
+            aodContent1.add(microOps.get(dest));
+        } else if (j != null && j.intValue() != k.intValue()) {
+            // merge the j component into the k component
+            List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
+            List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent2 = metaAsterixOpSkeletons.get(j);
+            aodContent1.addAll(aodContent2);
+            metaAsterixOpSkeletons.remove(j);
+            for (ILogicalOperator m : algebraicOpBelongingToMetaAsterixOp.keySet()) {
+                Integer g = algebraicOpBelongingToMetaAsterixOp.get(m);
+                if (g.intValue() == j.intValue()) {
+                    algebraicOpBelongingToMetaAsterixOp.put(m, k);
+                }
+            }
+        }
+
+    }
+
+    private int createNewMetaOpInfo(ILogicalOperator aop) {
+        int n = aodCounter;
+        aodCounter++;
+        List<Pair<IPushRuntimeFactory, RecordDescriptor>> metaOpContents = new ArrayList<Pair<IPushRuntimeFactory, RecordDescriptor>>();
+        metaOpContents.add(microOps.get(aop));
+        metaAsterixOpSkeletons.put(n, metaOpContents);
+        algebraicOpBelongingToMetaAsterixOp.put(aop, n);
+        return n;
+    }
+
+    private <E> void addAtPos(ArrayList<E> a, E elem, int pos) {
+        int n = a.size();
+        if (n > pos) {
+            a.set(pos, elem);
+        } else {
+            for (int k = n; k < pos; k++) {
+                a.add(null);
+            }
+            a.add(elem);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
new file mode 100644
index 0000000..a181304
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFamilyProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
+
+public class JobGenContext {
+    private final IOperatorSchema outerFlowSchema;
+    private final Map<ILogicalOperator, IOperatorSchema> schemaMap = new HashMap<ILogicalOperator, IOperatorSchema>();
+    private final ISerializerDeserializerProvider serializerDeserializerProvider;
+    private final IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
+    private final IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider;
+    private final IBinaryComparatorFactoryProvider comparatorFactoryProvider;
+    private final IPrinterFactoryProvider printerFactoryProvider;
+    private final ITypeTraitProvider typeTraitProvider;
+    private final IMetadataProvider<?, ?> metadataProvider;
+    private final INullWriterFactory nullWriterFactory;
+    private final INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
+    private final Object appContext;
+    private final IBinaryBooleanInspectorFactory booleanInspectorFactory;
+    private final IBinaryIntegerInspectorFactory integerInspectorFactory;
+    private final IExpressionRuntimeProvider expressionRuntimeProvider;
+    private final IExpressionTypeComputer expressionTypeComputer;
+    private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
+    private final IPartialAggregationTypeComputer partialAggregationTypeComputer;
+    private final IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider;
+    private final int frameSize;
+    private AlgebricksPartitionConstraint clusterLocations;
+    private int varCounter;
+    private final ITypingContext typingContext;
+
+    public JobGenContext(IOperatorSchema outerFlowSchema, IMetadataProvider<?, ?> metadataProvider, Object appContext,
+            ISerializerDeserializerProvider serializerDeserializerProvider,
+            IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider,
+            IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider,
+            IBinaryComparatorFactoryProvider comparatorFactoryProvider, ITypeTraitProvider typeTraitProvider,
+            IBinaryBooleanInspectorFactory booleanInspectorFactory,
+            IBinaryIntegerInspectorFactory integerInspectorFactory, IPrinterFactoryProvider printerFactoryProvider,
+            INullWriterFactory nullWriterFactory,
+            INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider,
+            IExpressionRuntimeProvider expressionRuntimeProvider, IExpressionTypeComputer expressionTypeComputer,
+            INullableTypeComputer nullableTypeComputer, ITypingContext typingContext,
+            IExpressionEvalSizeComputer expressionEvalSizeComputer,
+            IPartialAggregationTypeComputer partialAggregationTypeComputer,
+            IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider, int frameSize,
+            AlgebricksPartitionConstraint clusterLocations) {
+        this.outerFlowSchema = outerFlowSchema;
+        this.metadataProvider = metadataProvider;
+        this.appContext = appContext;
+        this.serializerDeserializerProvider = serializerDeserializerProvider;
+        this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
+        this.hashFunctionFamilyProvider = hashFunctionFamilyProvider;
+        this.comparatorFactoryProvider = comparatorFactoryProvider;
+        this.typeTraitProvider = typeTraitProvider;
+        this.booleanInspectorFactory = booleanInspectorFactory;
+        this.integerInspectorFactory = integerInspectorFactory;
+        this.printerFactoryProvider = printerFactoryProvider;
+        this.clusterLocations = clusterLocations;
+        this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
+        this.nullWriterFactory = nullWriterFactory;
+        this.expressionRuntimeProvider = expressionRuntimeProvider;
+        this.expressionTypeComputer = expressionTypeComputer;
+        this.typingContext = typingContext;
+        this.expressionEvalSizeComputer = expressionEvalSizeComputer;
+        this.partialAggregationTypeComputer = partialAggregationTypeComputer;
+        this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
+        this.frameSize = frameSize;
+        this.varCounter = 0;
+    }
+
+    public IOperatorSchema getOuterFlowSchema() {
+        return outerFlowSchema;
+    }
+
+    public AlgebricksPartitionConstraint getClusterLocations() {
+        return clusterLocations;
+    }
+
+    public IMetadataProvider<?, ?> getMetadataProvider() {
+        return metadataProvider;
+    }
+
+    public Object getAppContext() {
+        return appContext;
+    }
+
+    public ISerializerDeserializerProvider getSerializerDeserializerProvider() {
+        return serializerDeserializerProvider;
+    }
+
+    public IBinaryHashFunctionFactoryProvider getBinaryHashFunctionFactoryProvider() {
+        return hashFunctionFactoryProvider;
+    }
+
+    public IBinaryHashFunctionFamilyProvider getBinaryHashFunctionFamilyProvider() {
+        return hashFunctionFamilyProvider;
+    }
+
+    public IBinaryComparatorFactoryProvider getBinaryComparatorFactoryProvider() {
+        return comparatorFactoryProvider;
+    }
+
+    public ITypeTraitProvider getTypeTraitProvider() {
+        return typeTraitProvider;
+    }
+
+    public IBinaryBooleanInspectorFactory getBinaryBooleanInspectorFactory() {
+        return booleanInspectorFactory;
+    }
+
+    public IBinaryIntegerInspectorFactory getBinaryIntegerInspectorFactory() {
+        return integerInspectorFactory;
+    }
+
+    public IPrinterFactoryProvider getPrinterFactoryProvider() {
+        return printerFactoryProvider;
+    }
+
+    public IPredicateEvaluatorFactoryProvider getPredicateEvaluatorFactoryProvider() {
+        return predEvaluatorFactoryProvider;
+    }
+
+    public IExpressionRuntimeProvider getExpressionRuntimeProvider() {
+        return expressionRuntimeProvider;
+    }
+
+    public IOperatorSchema getSchema(ILogicalOperator op) {
+        return schemaMap.get(op);
+    }
+
+    public void putSchema(ILogicalOperator op, IOperatorSchema schema) {
+        schemaMap.put(op, schema);
+    }
+
+    public LogicalVariable createNewVar() {
+        varCounter++;
+        LogicalVariable var = new LogicalVariable(-varCounter);
+        return var;
+    }
+
+    public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env) throws AlgebricksException {
+        return expressionTypeComputer.getType(expr, typingContext.getMetadataProvider(), env);
+    }
+
+    public INullWriterFactory getNullWriterFactory() {
+        return nullWriterFactory;
+    }
+
+    public INormalizedKeyComputerFactoryProvider getNormalizedKeyComputerFactoryProvider() {
+        return normalizedKeyComputerFactoryProvider;
+    }
+
+    public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
+        return expressionEvalSizeComputer;
+    }
+
+    public int getFrameSize() {
+        return frameSize;
+    }
+
+    public IPartialAggregationTypeComputer getPartialAggregationTypeComputer() {
+        return partialAggregationTypeComputer;
+    }
+
+    public IVariableTypeEnvironment getTypeEnvironment(ILogicalOperator op) {
+        return typingContext.getOutputTypeEnvironment(op);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
new file mode 100644
index 0000000..2d195e4
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
@@ -0,0 +1,182 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFamilyProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public final class JobGenHelper {
+
+    private static final Logger LOGGER = Logger.getLogger(JobGenHelper.class.getName());
+
+    @SuppressWarnings("rawtypes")
+    public static RecordDescriptor mkRecordDescriptor(IVariableTypeEnvironment env, IOperatorSchema opSchema,
+            JobGenContext context) throws AlgebricksException {
+        ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()];
+        ITypeTraits[] typeTraits = new ITypeTraits[opSchema.getSize()];
+        ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider();
+        ITypeTraitProvider ttp = context.getTypeTraitProvider();
+        int i = 0;
+        for (LogicalVariable var : opSchema) {
+            Object t = env.getVarType(var);
+            if (t == null) {
+                LOGGER.warning("No type for variable " + var);
+            }
+            fields[i] = sdp.getSerializerDeserializer(t);
+            typeTraits[i] = ttp.getTypeTrait(t);
+            i++;
+        }
+        return new RecordDescriptor(fields, typeTraits);
+    }
+
+    public static IPrinterFactory[] mkPrinterFactories(IOperatorSchema opSchema, IVariableTypeEnvironment env,
+            JobGenContext context, int[] printColumns) throws AlgebricksException {
+        IPrinterFactory[] pf = new IPrinterFactory[printColumns.length];
+        IPrinterFactoryProvider pff = context.getPrinterFactoryProvider();
+        for (int i = 0; i < pf.length; i++) {
+            LogicalVariable v = opSchema.getVariable(printColumns[i]);
+            Object t = env.getVarType(v);
+            pf[i] = pff.getPrinterFactory(t);
+        }
+        return pf;
+    }
+
+    public static int[] variablesToFieldIndexes(Collection<LogicalVariable> varLogical, IOperatorSchema opSchema) {
+        int[] tuplePos = new int[varLogical.size()];
+        int i = 0;
+        for (LogicalVariable var : varLogical) {
+            tuplePos[i] = opSchema.findVariable(var);
+            i++;
+        }
+        return tuplePos;
+    }
+
+    public static IBinaryHashFunctionFactory[] variablesToBinaryHashFunctionFactories(
+            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+            throws AlgebricksException {
+        IBinaryHashFunctionFactory[] funFactories = new IBinaryHashFunctionFactory[varLogical.size()];
+        int i = 0;
+        IBinaryHashFunctionFactoryProvider bhffProvider = context.getBinaryHashFunctionFactoryProvider();
+        for (LogicalVariable var : varLogical) {
+            Object type = env.getVarType(var);
+            funFactories[i++] = bhffProvider.getBinaryHashFunctionFactory(type);
+        }
+        return funFactories;
+    }
+
+    public static IBinaryHashFunctionFamily[] variablesToBinaryHashFunctionFamilies(
+            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+            throws AlgebricksException {
+        IBinaryHashFunctionFamily[] funFamilies = new IBinaryHashFunctionFamily[varLogical.size()];
+        int i = 0;
+        IBinaryHashFunctionFamilyProvider bhffProvider = context.getBinaryHashFunctionFamilyProvider();
+        for (LogicalVariable var : varLogical) {
+            Object type = env.getVarType(var);
+            funFamilies[i++] = bhffProvider.getBinaryHashFunctionFamily(type);
+        }
+        return funFamilies;
+    }
+
+    public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(
+            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+            throws AlgebricksException {
+        IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[varLogical.size()];
+        IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
+        int i = 0;
+        for (LogicalVariable v : varLogical) {
+            Object type = env.getVarType(v);
+            compFactories[i++] = bcfProvider.getBinaryComparatorFactory(type, true);
+        }
+        return compFactories;
+    }
+
+    public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(List<LogicalVariable> varLogical,
+            int start, int size, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
+        IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[size];
+        IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
+        for (int i = 0; i < size; i++) {
+            Object type = env.getVarType(varLogical.get(start + i));
+            compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true);
+        }
+        return compFactories;
+    }
+
+    public static INormalizedKeyComputerFactory variablesToAscNormalizedKeyComputerFactory(
+            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+            throws AlgebricksException {
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        if (nkcfProvider == null)
+            return null;
+        for (LogicalVariable v : varLogical) {
+            Object type = env.getVarType(v);
+            return nkcfProvider.getNormalizedKeyComputerFactory(type, true);
+        }
+        return null;
+    }
+
+    public static ITypeTraits[] variablesToTypeTraits(Collection<LogicalVariable> varLogical,
+            IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
+        ITypeTraits[] typeTraits = new ITypeTraits[varLogical.size()];
+        ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
+        int i = 0;
+        for (LogicalVariable v : varLogical) {
+            Object type = env.getVarType(v);
+            typeTraits[i++] = typeTraitProvider.getTypeTrait(type);
+        }
+        return typeTraits;
+    }
+
+    public static ITypeTraits[] variablesToTypeTraits(List<LogicalVariable> varLogical, int start, int size,
+            IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
+        ITypeTraits[] typeTraits = new ITypeTraits[size];
+        ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
+        for (int i = 0; i < size; i++) {
+            Object type = env.getVarType(varLogical.get(start + i));
+            typeTraits[i] = typeTraitProvider.getTypeTrait(type);
+        }
+        return typeTraits;
+    }
+
+    public static int[] projectAllVariables(IOperatorSchema opSchema) {
+        int[] projectionList = new int[opSchema.getSize()];
+        int k = 0;
+        for (LogicalVariable v : opSchema) {
+            projectionList[k++] = opSchema.findVariable(v);
+        }
+        return projectionList;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
new file mode 100644
index 0000000..dc1fb35
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+
+public class OperatorSchemaImpl implements IOperatorSchema {
+    private final Map<LogicalVariable, Integer> varMap;
+
+    private final List<LogicalVariable> varList;
+
+    public OperatorSchemaImpl() {
+        varMap = new HashMap<LogicalVariable, Integer>();
+        varList = new ArrayList<LogicalVariable>();
+    }
+
+    @Override
+    public void addAllVariables(IOperatorSchema source) {
+        for (LogicalVariable v : source) {
+            varMap.put(v, varList.size());
+            varList.add(v);
+        }
+    }
+
+    @Override
+    public void addAllNewVariables(IOperatorSchema source) {
+        for (LogicalVariable v : source) {
+            if (varMap.get(v) == null) {
+                varMap.put(v, varList.size());
+                varList.add(v);
+            }
+        }
+    }
+
+    @Override
+    public int addVariable(LogicalVariable var) {
+        int idx = varList.size();
+        varMap.put(var, idx);
+        varList.add(var);
+        return idx;
+    }
+
+    @Override
+    public void clear() {
+        varMap.clear();
+        varList.clear();
+    }
+
+    @Override
+    public int findVariable(LogicalVariable var) {
+        Integer i = varMap.get(var);
+        if (i == null) {
+            return -1;
+        }
+        return i;
+    }
+
+    @Override
+    public int getSize() {
+        return varList.size();
+    }
+
+    @Override
+    public LogicalVariable getVariable(int index) {
+        return varList.get(index);
+    }
+
+    @Override
+    public Iterator<LogicalVariable> iterator() {
+        return varList.iterator();
+    }
+
+    @Override
+    public String toString() {
+        return varMap.toString();
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
new file mode 100644
index 0000000..3af57ad
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/PlanCompiler.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class PlanCompiler {
+    private JobGenContext context;
+    private Map<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> operatorVisitedToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
+
+    public PlanCompiler(JobGenContext context) {
+        this.context = context;
+    }
+
+    public JobGenContext getContext() {
+        return context;
+    }
+
+    public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema,
+            IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException {
+        JobSpecification spec = new JobSpecification(context.getFrameSize());
+        if (jobEventListenerFactory != null) {
+            spec.setJobletEventListenerFactory(jobEventListenerFactory);
+        }
+        List<ILogicalOperator> rootOps = new ArrayList<ILogicalOperator>();
+        IHyracksJobBuilder builder = new JobBuilder(spec, context.getClusterLocations());
+        for (Mutable<ILogicalOperator> opRef : plan.getRoots()) {
+            compileOpRef(opRef, spec, builder, outerPlanSchema);
+            rootOps.add(opRef.getValue());
+        }
+        reviseEdges(builder);
+        operatorVisitedToParents.clear();
+        builder.buildSpec(rootOps);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        // Do not do activity cluster planning because it is slow on large clusters
+        spec.setUseConnectorPolicyForScheduling(false);
+        return spec;
+    }
+
+    private void compileOpRef(Mutable<ILogicalOperator> opRef, IOperatorDescriptorRegistry spec,
+            IHyracksJobBuilder builder, IOperatorSchema outerPlanSchema) throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
+        int n = op.getInputs().size();
+        IOperatorSchema[] schemas = new IOperatorSchema[n];
+        int i = 0;
+        for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
+            List<Mutable<ILogicalOperator>> parents = operatorVisitedToParents.get(opRef2);
+            if (parents == null) {
+                parents = new ArrayList<Mutable<ILogicalOperator>>();
+                operatorVisitedToParents.put(opRef2, parents);
+                parents.add(opRef);
+                compileOpRef(opRef2, spec, builder, outerPlanSchema);
+                schemas[i++] = context.getSchema(opRef2.getValue());
+            } else {
+                if (!parents.contains(opRef))
+                    parents.add(opRef);
+                schemas[i++] = context.getSchema(opRef2.getValue());
+                continue;
+            }
+        }
+
+        IOperatorSchema opSchema = new OperatorSchemaImpl();
+        context.putSchema(op, opSchema);
+        op.getVariablePropagationPolicy().propagateVariables(opSchema, schemas);
+        op.contributeRuntimeOperator(builder, context, opSchema, schemas, outerPlanSchema);
+    }
+
+    private void reviseEdges(IHyracksJobBuilder builder) {
+        /**
+         * revise the edges for the case of replicate operator
+         */
+        for (Entry<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> entry : operatorVisitedToParents
+                .entrySet()) {
+            Mutable<ILogicalOperator> child = entry.getKey();
+            List<Mutable<ILogicalOperator>> parents = entry.getValue();
+            if (parents.size() > 1) {
+                if (child.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) {
+                    ReplicateOperator rop = (ReplicateOperator) child.getValue();
+                    if (rop.isBlocker()) {
+                        // make the order of the graph edges consistent with the order of rop's outputs
+                        List<Mutable<ILogicalOperator>> outputs = rop.getOutputs();
+                        for (Mutable<ILogicalOperator> parent : parents) {
+                            builder.contributeGraphEdge(child.getValue(), outputs.indexOf(parent), parent.getValue(), 0);
+                        }
+                    } else {
+                        int i = 0;
+                        for (Mutable<ILogicalOperator> parent : parents) {
+                            builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0);
+                            i++;
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
new file mode 100644
index 0000000..e7b469a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.Collection;
+import java.util.logging.Level;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public abstract class AbstractRuleController {
+
+    protected IOptimizationContext context;
+
+    public AbstractRuleController() {
+    }
+
+    public void setContext(IOptimizationContext context) {
+        this.context = context;
+    }
+
+    /**
+     * Each rewriting strategy may differ in the
+     * 
+     * @param root
+     * @param ruleClasses
+     * @return true iff one of the rules in the collection fired
+     */
+    public abstract boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
+            Collection<IAlgebraicRewriteRule> rules) throws AlgebricksException;
+
+    /**
+     * @param opRef
+     * @param rule
+     * @return true if any rewrite was fired, either on opRef or any operator
+     *         under it.
+     */
+    protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule)
+            throws AlgebricksException {
+        return rewriteOperatorRef(opRef, rule, true, false);
+    }
+
+    private String getPlanString(Mutable<ILogicalOperator> opRef) throws AlgebricksException {
+        if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
+            StringBuilder sb = new StringBuilder();
+            LogicalOperatorPrettyPrintVisitor pvisitor = context.getPrettyPrintVisitor();
+            PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
+            return sb.toString();
+        }
+        return null;
+    }
+
+    private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan)
+            throws AlgebricksException {
+        if (AlgebricksConfig.ALGEBRICKS_LOGGER.isLoggable(Level.FINE)) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Rule " + rule.getClass() + " fired.\n");
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Before plan\n" + beforePlan + "\n");
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> After plan\n" + afterPlan + "\n");
+        }
+    }
+
+    protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule,
+            boolean enterNestedPlans, boolean fullDFS) throws AlgebricksException {
+
+        String preBeforePlan = getPlanString(opRef);
+        if (rule.rewritePre(opRef, context)) {
+            String preAfterPlan = getPlanString(opRef);
+            printRuleApplication(rule, preBeforePlan, preAfterPlan);
+            return true;
+        }
+        boolean rewritten = false;
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+        for (Mutable<ILogicalOperator> inp : op.getInputs()) {
+            if (rewriteOperatorRef(inp, rule, enterNestedPlans, fullDFS)) {
+                rewritten = true;
+                if (!fullDFS) {
+                    break;
+                }
+            }
+        }
+
+        if (op.hasNestedPlans() && enterNestedPlans) {
+            AbstractOperatorWithNestedPlans o2 = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : o2.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    if (rewriteOperatorRef(r, rule, enterNestedPlans, fullDFS)) {
+                        rewritten = true;
+                        if (!fullDFS) {
+                            break;
+                        }
+                    }
+                }
+                if (rewritten && !fullDFS) {
+                    break;
+                }
+            }
+        }
+
+        String postBeforePlan = getPlanString(opRef);
+        if (rule.rewritePost(opRef, context)) {
+            String postAfterPlan = getPlanString(opRef);
+            printRuleApplication(rule, postBeforePlan, postAfterPlan);
+            return true;
+        }
+
+        return rewritten;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
new file mode 100644
index 0000000..361dad3
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+
+public class AlgebricksOptimizationContext implements IOptimizationContext {
+
+    private int varCounter;
+    private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
+    private final IMergeAggregationExpressionFactory mergeAggregationExpressionFactory;
+    private final PhysicalOptimizationConfig physicalOptimizationConfig;
+    private final IVariableEvalSizeEnvironment varEvalSizeEnv = new IVariableEvalSizeEnvironment() {
+
+        Map<LogicalVariable, Integer> varSizeMap = new HashMap<LogicalVariable, Integer>();
+
+        @Override
+        public void setVariableEvalSize(LogicalVariable var, int size) {
+            varSizeMap.put(var, size);
+        }
+
+        @Override
+        public int getVariableEvalSize(LogicalVariable var) {
+            return varSizeMap.get(var);
+        }
+    };
+
+    private Map<ILogicalOperator, IVariableTypeEnvironment> typeEnvMap = new HashMap<ILogicalOperator, IVariableTypeEnvironment>();
+
+    private Map<ILogicalOperator, HashSet<ILogicalOperator>> alreadyCompared = new HashMap<ILogicalOperator, HashSet<ILogicalOperator>>();
+    private Map<IAlgebraicRewriteRule, HashSet<ILogicalOperator>> dontApply = new HashMap<IAlgebraicRewriteRule, HashSet<ILogicalOperator>>();
+    private Map<LogicalVariable, FunctionalDependency> recordToPrimaryKey = new HashMap<LogicalVariable, FunctionalDependency>();
+
+    @SuppressWarnings("unchecked")
+    private IMetadataProvider metadataProvider;
+    private HashSet<LogicalVariable> notToBeInlinedVars = new HashSet<LogicalVariable>();
+
+    protected final Map<ILogicalOperator, List<FunctionalDependency>> fdGlobalMap = new HashMap<ILogicalOperator, List<FunctionalDependency>>();
+    protected final Map<ILogicalOperator, Map<LogicalVariable, EquivalenceClass>> eqClassGlobalMap = new HashMap<ILogicalOperator, Map<LogicalVariable, EquivalenceClass>>();
+
+    protected final Map<ILogicalOperator, ILogicalPropertiesVector> logicalProps = new HashMap<ILogicalOperator, ILogicalPropertiesVector>();
+    private final IExpressionTypeComputer expressionTypeComputer;
+    private final INullableTypeComputer nullableTypeComputer;
+    private final LogicalOperatorPrettyPrintVisitor prettyPrintVisitor;
+
+    public AlgebricksOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
+            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+            IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+            PhysicalOptimizationConfig physicalOptimizationConfig) {
+        this(varCounter, expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
+                nullableTypeComputer, physicalOptimizationConfig, new LogicalOperatorPrettyPrintVisitor());
+    }
+
+    public AlgebricksOptimizationContext(int varCounter, IExpressionEvalSizeComputer expressionEvalSizeComputer,
+            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+            IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+            PhysicalOptimizationConfig physicalOptimizationConfig, LogicalOperatorPrettyPrintVisitor prettyPrintVisitor) {
+        this.varCounter = varCounter;
+        this.expressionEvalSizeComputer = expressionEvalSizeComputer;
+        this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
+        this.expressionTypeComputer = expressionTypeComputer;
+        this.nullableTypeComputer = nullableTypeComputer;
+        this.physicalOptimizationConfig = physicalOptimizationConfig;
+        this.prettyPrintVisitor = prettyPrintVisitor;
+    }
+
+    public int getVarCounter() {
+        return varCounter;
+    }
+
+    public void setVarCounter(int varCounter) {
+        this.varCounter = varCounter;
+    }
+
+    public LogicalVariable newVar() {
+        varCounter++;
+        LogicalVariable var = new LogicalVariable(varCounter);
+        return var;
+    }
+
+    @SuppressWarnings("unchecked")
+    public IMetadataProvider getMetadataProvider() {
+        return metadataProvider;
+    }
+
+    public void setMetadataDeclarations(IMetadataProvider<?, ?> metadataProvider) {
+        this.metadataProvider = metadataProvider;
+    }
+
+    public boolean checkIfInDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op) {
+        HashSet<ILogicalOperator> operators = dontApply.get(rule);
+        if (operators == null) {
+            return false;
+        } else {
+            return operators.contains(op);
+        }
+    }
+
+    public void addToDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op) {
+        HashSet<ILogicalOperator> operators = dontApply.get(rule);
+        if (operators == null) {
+            HashSet<ILogicalOperator> os = new HashSet<ILogicalOperator>();
+            os.add(op);
+            dontApply.put(rule, os);
+        } else {
+            operators.add(op);
+        }
+
+    }
+
+    /*
+     * returns true if op1 and op2 have already been compared
+     */
+    @Override
+    public boolean checkAndAddToAlreadyCompared(ILogicalOperator op1, ILogicalOperator op2) {
+        HashSet<ILogicalOperator> ops = alreadyCompared.get(op1);
+        if (ops == null) {
+            HashSet<ILogicalOperator> newEntry = new HashSet<ILogicalOperator>();
+            newEntry.add(op2);
+            alreadyCompared.put(op1, newEntry);
+            return false;
+        } else {
+            if (ops.contains(op2)) {
+                return true;
+            } else {
+                ops.add(op2);
+                return false;
+            }
+        }
+    }
+    
+    @Override
+    public void removeFromAlreadyCompared(ILogicalOperator op1) {
+        alreadyCompared.remove(op1);
+    }
+
+    public void addNotToBeInlinedVar(LogicalVariable var) {
+        notToBeInlinedVars.add(var);
+    }
+
+    public boolean shouldNotBeInlined(LogicalVariable var) {
+        return notToBeInlinedVars.contains(var);
+    }
+
+    public void addPrimaryKey(FunctionalDependency pk) {
+        assert (pk.getTail().size() == 1);
+        LogicalVariable recordVar = pk.getTail().get(0);
+        recordToPrimaryKey.put(recordVar, pk);
+    }
+
+    public List<LogicalVariable> findPrimaryKey(LogicalVariable recordVar) {
+        FunctionalDependency fd = recordToPrimaryKey.get(recordVar);
+        if (fd == null) {
+            return null;
+        }
+        return fd.getHead();
+    }
+
+    @Override
+    public Map<LogicalVariable, EquivalenceClass> getEquivalenceClassMap(ILogicalOperator op) {
+        return eqClassGlobalMap.get(op);
+    }
+
+    @Override
+    public List<FunctionalDependency> getFDList(ILogicalOperator op) {
+        return fdGlobalMap.get(op);
+    }
+
+    @Override
+    public void putEquivalenceClassMap(ILogicalOperator op, Map<LogicalVariable, EquivalenceClass> eqClassMap) {
+        this.eqClassGlobalMap.put(op, eqClassMap);
+    }
+
+    @Override
+    public void putFDList(ILogicalOperator op, List<FunctionalDependency> fdList) {
+        this.fdGlobalMap.put(op, fdList);
+    }
+
+    @Override
+    public ILogicalPropertiesVector getLogicalPropertiesVector(ILogicalOperator op) {
+        return logicalProps.get(op);
+    }
+
+    @Override
+    public void putLogicalPropertiesVector(ILogicalOperator op, ILogicalPropertiesVector v) {
+        logicalProps.put(op, v);
+    }
+
+    @Override
+    public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
+        return expressionEvalSizeComputer;
+    }
+
+    @Override
+    public IVariableEvalSizeEnvironment getVariableEvalSizeEnvironment() {
+        return varEvalSizeEnv;
+    }
+
+    public IMergeAggregationExpressionFactory getMergeAggregationExpressionFactory() {
+        return mergeAggregationExpressionFactory;
+    }
+
+    public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
+        return physicalOptimizationConfig;
+    }
+
+    @Override
+    public IVariableTypeEnvironment getOutputTypeEnvironment(ILogicalOperator op) {
+        return typeEnvMap.get(op);
+    }
+
+    @Override
+    public void setOutputTypeEnvironment(ILogicalOperator op, IVariableTypeEnvironment env) {
+        typeEnvMap.put(op, env);
+    }
+
+    @Override
+    public IExpressionTypeComputer getExpressionTypeComputer() {
+        return expressionTypeComputer;
+    }
+
+    @Override
+    public INullableTypeComputer getNullableTypeComputer() {
+        return nullableTypeComputer;
+    }
+
+    @Override
+    public void invalidateTypeEnvironmentForOperator(ILogicalOperator op) {
+        typeEnvMap.put(op, null);
+    }
+
+    @Override
+    public void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException {
+        setOutputTypeEnvironment(op, op.computeOutputTypeEnvironment(this));
+    }
+
+    @Override
+    public void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars) {
+        for (Map.Entry<LogicalVariable, FunctionalDependency> me : recordToPrimaryKey.entrySet()) {
+            FunctionalDependency fd = me.getValue();
+            List<LogicalVariable> hd = new ArrayList<LogicalVariable>();
+            for (LogicalVariable v : fd.getHead()) {
+                LogicalVariable v2 = mappedVars.get(v);
+                if (v2 == null) {
+                    hd.add(v);
+                } else {
+                    hd.add(v2);
+                }
+            }
+            List<LogicalVariable> tl = new ArrayList<LogicalVariable>();
+            for (LogicalVariable v : fd.getTail()) {
+                LogicalVariable v2 = mappedVars.get(v);
+                if (v2 == null) {
+                    tl.add(v);
+                } else {
+                    tl.add(v2);
+                }
+            }
+            me.setValue(new FunctionalDependency(hd, tl));
+        }
+    }
+    
+    @Override
+    public LogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor() {
+        return prettyPrintVisitor;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
new file mode 100644
index 0000000..17c6900
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class HeuristicOptimizer {
+
+    public static PhysicalOperatorTag[] hyraxOperators = new PhysicalOperatorTag[] {
+            PhysicalOperatorTag.DATASOURCE_SCAN, PhysicalOperatorTag.BTREE_SEARCH,
+            PhysicalOperatorTag.EXTERNAL_GROUP_BY, PhysicalOperatorTag.HASH_GROUP_BY, PhysicalOperatorTag.HDFS_READER,
+            PhysicalOperatorTag.HYBRID_HASH_JOIN, PhysicalOperatorTag.IN_MEMORY_HASH_JOIN,
+            PhysicalOperatorTag.NESTED_LOOP, PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY,
+            PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY, PhysicalOperatorTag.SPLIT, PhysicalOperatorTag.STABLE_SORT,
+            PhysicalOperatorTag.UNION_ALL };
+    public static PhysicalOperatorTag[] hyraxOperatorsBelowWhichJobGenIsDisabled = new PhysicalOperatorTag[] {};
+
+    public static boolean isHyraxOp(PhysicalOperatorTag opTag) {
+        for (PhysicalOperatorTag t : hyraxOperators) {
+            if (t == opTag) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private IOptimizationContext context;
+    private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
+    private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
+    private ILogicalPlan plan;
+
+    public HeuristicOptimizer(ILogicalPlan plan,
+            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites,
+            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites,
+            IOptimizationContext context) {
+        this.plan = plan;
+        this.context = context;
+        this.logicalRewrites = logicalRewrites;
+        this.physicalRewrites = physicalRewrites;
+    }
+
+    public void optimize() throws AlgebricksException {
+        if (plan == null) {
+            return;
+        }
+        if (AlgebricksConfig.DEBUG) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting logical optimizations.\n");
+        }
+
+        StringBuilder sb = new StringBuilder();
+        PlanPrettyPrinter.printPlan(plan, sb, context.getPrettyPrintVisitor(), 0);
+        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Logical Plan:\n" + sb.toString());
+        runOptimizationSets(plan, logicalRewrites);
+        computeSchemaBottomUpForPlan(plan);
+        runPhysicalOptimizations(plan, physicalRewrites);
+        StringBuilder sb2 = new StringBuilder();
+        PlanPrettyPrinter.printPlan(plan, sb2, context.getPrettyPrintVisitor(), 0);
+        AlgebricksConfig.ALGEBRICKS_LOGGER.info("Optimized Plan:\n" + sb2.toString());
+    }
+
+    private void runOptimizationSets(ILogicalPlan plan,
+            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> optimSet) throws AlgebricksException {
+        for (Pair<AbstractRuleController, List<IAlgebraicRewriteRule>> ruleList : optimSet) {
+            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+                ruleList.first.setContext(context);
+                ruleList.first.rewriteWithRuleCollection(r, ruleList.second);
+            }
+        }
+    }
+
+    private static void computeSchemaBottomUpForPlan(ILogicalPlan p) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
+            computeSchemaBottomUpForOp((AbstractLogicalOperator) r.getValue());
+        }
+    }
+
+    private static void computeSchemaBottomUpForOp(AbstractLogicalOperator op) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            computeSchemaBottomUpForOp((AbstractLogicalOperator) i.getValue());
+        }
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : a.getNestedPlans()) {
+                computeSchemaBottomUpForPlan(p);
+            }
+        }
+        op.recomputeSchema();
+    }
+
+    private void runPhysicalOptimizations(ILogicalPlan plan,
+            List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites)
+            throws AlgebricksException {
+        if (AlgebricksConfig.DEBUG) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting physical optimizations.\n");
+        }
+        // PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(plan);
+        runOptimizationSets(plan, physicalRewrites);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
new file mode 100644
index 0000000..6bf0b72
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+
+public interface IAlgebraicRewriteRule {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException;
+
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
new file mode 100644
index 0000000..70b7f64
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+
+public interface IOptimizationContextFactory {
+    public IOptimizationContext createOptimizationContext(int varCounter,
+            IExpressionEvalSizeComputer expressionEvalSizeComputer,
+            IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+            IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+            PhysicalOptimizationConfig physicalOptimizationConfig);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
new file mode 100644
index 0000000..47d2266
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.Properties;
+
+public class PhysicalOptimizationConfig {
+    private static final int MB = 1048576;
+    
+    private static final String FRAMESIZE = "FRAMESIZE";
+    private static final String MAX_FRAMES_EXTERNAL_SORT = "MAX_FRAMES_EXTERNAL_SORT";
+    private static final String MAX_FRAMES_EXTERNAL_GROUP_BY = "MAX_FRAMES_EXTERNAL_GROUP_BY";
+    private static final String MAX_FRAMES_LEFT_INPUT_HYBRID_HASH = "MAX_FRAMES_LEFT_INPUT_HYBRID_HASH";
+    private static final String MAX_FRAMES_HYBRID_HASH = "MAX_FRAMES_HYBRID_HASH";
+    private static final String FUDGE_FACTOR = "FUDGE_FACTOR";
+    private static final String MAX_RECORDS_PER_FRAME = "MAX_RECORDS_PER_FRAME";
+    
+    private static final String DEFAULT_HASH_GROUP_TABLE_SIZE = "DEFAULT_HASH_GROUP_TABLE_SIZE";
+    private static final String DEFAULT_EXTERNAL_GROUP_TABLE_SIZE = "DEFAULT_EXTERNAL_GROUP_TABLE_SIZE";
+    private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
+
+    private Properties properties = new Properties();
+
+    public PhysicalOptimizationConfig() {
+        int frameSize = 32768;
+        setInt(FRAMESIZE, frameSize);
+        setInt(MAX_FRAMES_EXTERNAL_SORT, (int) (((long) 32 * MB) / frameSize));
+        setInt(MAX_FRAMES_EXTERNAL_GROUP_BY, (int) (((long) 32 * MB) / frameSize));
+
+        // use http://www.rsok.com/~jrm/printprimes.html to find prime numbers
+        setInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
+        setInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, 10485767);
+        setInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, 10485767);
+    }
+
+    public int getFrameSize() {
+        return getInt(FRAMESIZE, 32768);
+    }
+
+    public void setFrameSize(int frameSize) {
+        setInt(FRAMESIZE, frameSize);
+    }
+    
+    public double getFudgeFactor() {
+        return getDouble(FUDGE_FACTOR, 1.3);
+    }
+
+    public void setFudgeFactor(double fudgeFactor) {
+        setDouble(FUDGE_FACTOR, fudgeFactor);
+    }
+    
+    public int getMaxRecordsPerFrame() {
+        return getInt(MAX_RECORDS_PER_FRAME, 512);
+    }
+
+    public void setMaxRecordsPerFrame(int maxRecords) {
+        setInt(MAX_RECORDS_PER_FRAME, maxRecords);
+    }
+
+    public int getMaxFramesLeftInputHybridHash() {
+        int frameSize = getFrameSize();
+        return getInt(MAX_FRAMES_LEFT_INPUT_HYBRID_HASH, (int) (140L * 1024 * MB / frameSize));
+    }
+
+    public void setMaxFramesLeftInputHybridHash(int frameLimit) {
+        setInt(MAX_FRAMES_LEFT_INPUT_HYBRID_HASH, frameLimit);
+    }
+    
+    public int getMaxFramesHybridHash() {
+        int frameSize = getFrameSize();
+        return getInt(MAX_FRAMES_HYBRID_HASH, (int) (64L * MB / frameSize));
+    }
+
+    public void setMaxFramesHybridHash(int frameLimit) {
+        setInt(MAX_FRAMES_HYBRID_HASH, frameLimit);
+    }
+
+    public int getMaxFramesExternalGroupBy() {
+        int frameSize = getFrameSize();
+        return getInt(MAX_FRAMES_EXTERNAL_GROUP_BY, (int) (((long) 256 * MB) / frameSize));
+    }
+
+    public void setMaxFramesExternalGroupBy(int frameLimit) {
+        setInt(MAX_FRAMES_EXTERNAL_GROUP_BY, frameLimit);
+    }
+    
+    public int getMaxFramesExternalSort() {
+        int frameSize = getFrameSize();
+        return getInt(MAX_FRAMES_EXTERNAL_SORT, (int) (((long) 32 * MB) / frameSize));
+    }
+
+    public void setMaxFramesExternalSort(int frameLimit) {
+        setInt(MAX_FRAMES_EXTERNAL_SORT, frameLimit);
+    }
+
+    public int getHashGroupByTableSize() {
+        return getInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
+    }
+
+    public void setHashGroupByTableSize(int tableSize) {
+        setInt(DEFAULT_HASH_GROUP_TABLE_SIZE, tableSize);
+    }
+
+    public int getExternalGroupByTableSize() {
+        return getInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, 10485767);
+    }
+
+    public void setExternalGroupByTableSize(int tableSize) {
+        setInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, tableSize);
+    }
+
+    public int getInMemHashJoinTableSize() {
+        return getInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, 10485767);
+    }
+
+    public void setInMemHashJoinTableSize(int tableSize) {
+        setInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, tableSize);
+    }
+
+    private void setInt(String property, int value) {
+        properties.setProperty(property, Integer.toString(value));
+    }
+
+    private int getInt(String property, int defaultValue) {
+        String value = properties.getProperty(property);
+        if (value == null)
+            return defaultValue;
+        else
+            return Integer.parseInt(value);
+    }
+    
+    private void setDouble(String property, double value) {
+        properties.setProperty(property, Double.toString(value));
+    }
+
+    private double getDouble(String property, double defaultValue) {
+        String value = properties.getProperty(property);
+        if (value == null)
+            return defaultValue;
+        else
+            return Double.parseDouble(value);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/Substitution.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/Substitution.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/Substitution.java
new file mode 100644
index 0000000..3ef4316
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/utils/Substitution.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.utils;
+
+public class Substitution<T> {
+    public T substituted;
+    public T substitutedWith;
+
+    public Substitution(T substituted, T substitutedWith) {
+        this.substituted = substituted;
+        this.substitutedWith = substitutedWith;
+    }
+}

[23/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java
new file mode 100644
index 0000000..85274de
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+
+public class PlanPlotter {
+
+    static Random randomGenerator = new Random();
+
+    public static void printLogicalPlan(ILogicalPlan plan) throws AlgebricksException {
+        int indent = 5;
+        StringBuilder out = new StringBuilder();
+        int randomInt = 10000 + randomGenerator.nextInt(100);
+        appendln(out, "digraph G {");
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            printVisualizationGraph((AbstractLogicalOperator) root.getValue(), indent, out, "", randomInt);
+        }
+        appendln(out, "\n}\n}");
+        try {
+            File file = File.createTempFile("logicalPlan", ".txt");
+            FileUtils.writeStringToFile(file, out.toString());
+            file.deleteOnExit();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void printOptimizedLogicalPlan(ILogicalPlan plan) throws AlgebricksException {
+        int indent = 5;
+        StringBuilder out = new StringBuilder();
+        int randomInt = 10000 + randomGenerator.nextInt(100);
+        appendln(out, "digraph G {");
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            printVisualizationGraph((AbstractLogicalOperator) root.getValue(), indent, out, "", randomInt);
+        }
+        appendln(out, "\n}\n}");
+        try {
+            File file = File.createTempFile("logicalOptimizedPlan", ".txt");
+            FileUtils.writeStringToFile(file, out.toString());
+            file.deleteOnExit();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /*
+     * DFS traversal function. Calls iteratively all children, and for each calls itself recursively
+     * Includes slim-maps only (no gathering of mappers to one)
+     */
+    public static void printVisualizationGraph(AbstractLogicalOperator op, int indent, StringBuilder out,
+            String current_supernode_name, int randomInt) {
+        if (!op.getInputs().isEmpty()) {
+            //String stringToVisualize = op.toStringForVisualizationGraph();
+            String stringToVisualize = op.getOperatorTag().name();
+            int firstOccurenceOf_ = stringToVisualize.indexOf("_");
+            String supernode_current = stringToVisualize.substring(firstOccurenceOf_ + 1, stringToVisualize.length());
+            if (current_supernode_name.isEmpty()) {
+                current_supernode_name = supernode_current;
+                appendln(out, new String("subgraph cluster_" + supernode_current + " {"));
+                pad(out, indent);
+                appendln(out, new String("node [style=filled, color = lightgray];"));
+                pad(out, indent);
+                appendln(out, new String("color=blue;"));
+                pad(out, indent);
+                String op_id = op.toString().substring(op.toString().indexOf("@") + 1, op.toString().length());
+                appendln(out, new String("label = \"" + supernode_current + "ID" + op_id + "\";"));
+                pad(out, indent);
+            }
+
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                String op_id = i.toString().substring(i.toString().indexOf("@") + 1, i.toString().length());
+                String logOpStr = ((AbstractLogicalOperator) i.getValue()).getOperatorTag().name() + "ID" + op_id;
+                firstOccurenceOf_ = logOpStr.indexOf("_");
+                String supernode_child = logOpStr.substring(firstOccurenceOf_ + 1, logOpStr.length());
+                if (!supernode_current.equals(supernode_child)) {
+                    appendln(out, new String("node [style=filled, color = lightgray];"));
+                    pad(out, indent);
+                    appendln(out, new String("color=blue"));
+                    pad(out, indent);
+                    appendln(out, new String("label = \"" + supernode_child + "\";"));
+                    pad(out, indent);
+                }
+
+                op_id = op.toString().substring(op.toString().indexOf("@") + 1, op.toString().length());
+                appendln(out, stringToVisualize + "ID" + op_id + "[style = filled]");
+                AbstractLogicalOperator child = (AbstractLogicalOperator) i.getValue();
+
+                pad(out, indent);
+                String op_id1 = op.toString().substring(op.toString().indexOf("@") + 1, op.toString().length());
+                append(out, op.getOperatorTag().name() + "ID" + op_id1 + " -> ");
+                String opc_id = child.toString()
+                        .substring(child.toString().indexOf("@") + 1, child.toString().length());
+                appendln(out, child.getOperatorTag().name() + "ID" + opc_id);
+
+                printVisualizationGraph(child, indent, out, supernode_current, (randomGenerator.nextInt(100) + 10000));
+
+            }
+        }
+
+    }
+
+    private static void appendln(StringBuilder buf, String s) {
+        buf.append(s);
+        buf.append("\n");
+    }
+
+    private static void append(StringBuilder buf, String s) {
+        buf.append(s);
+    }
+
+    private static void pad(StringBuilder buf, int indent) {
+        for (int i = 0; i < indent; ++i) {
+            buf.append(' ');
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
new file mode 100644
index 0000000..33c106b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+
+public class PlanPrettyPrinter {
+    public static void printPlan(ILogicalPlan plan, StringBuilder out, LogicalOperatorPrettyPrintVisitor pvisitor,
+            int indent) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            printOperator((AbstractLogicalOperator) root.getValue(), out, pvisitor, indent);
+        }
+    }
+
+    public static void printPhysicalOps(ILogicalPlan plan, StringBuilder out, int indent) {
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            printPhysicalOperator((AbstractLogicalOperator) root.getValue(), indent, out);
+        }
+    }
+
+    public static void printOperator(AbstractLogicalOperator op, StringBuilder out,
+            LogicalOperatorPrettyPrintVisitor pvisitor, int indent) throws AlgebricksException {
+        out.append(op.accept(pvisitor, indent));
+        IPhysicalOperator pOp = op.getPhysicalOperator();
+
+        if (pOp != null) {
+            out.append("\n");
+            pad(out, indent);
+            appendln(out, "-- " + pOp.toString() + "  |" + op.getExecutionMode() + "|");
+        } else {
+            appendln(out, " -- |" + op.getExecutionMode() + "|");
+        }
+
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            printOperator((AbstractLogicalOperator) i.getValue(), out, pvisitor, indent + 2);
+        }
+
+    }
+
+    public static void printPhysicalOperator(AbstractLogicalOperator op, int indent, StringBuilder out) {
+        IPhysicalOperator pOp = op.getPhysicalOperator();
+        pad(out, indent);
+        appendln(out, "-- " + pOp.toString() + "  |" + op.getExecutionMode() + "|");
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans opNest = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : opNest.getNestedPlans()) {
+                pad(out, indent + 8);
+                appendln(out, "{");
+                printPhysicalOps(p, out, indent + 10);
+                pad(out, indent + 8);
+                appendln(out, "}");
+            }
+        }
+
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            printPhysicalOperator((AbstractLogicalOperator) i.getValue(), indent + 2, out);
+        }
+
+    }
+
+    private static void appendln(StringBuilder buf, String s) {
+        buf.append(s);
+        buf.append("\n");
+    }
+
+    private static void pad(StringBuilder buf, int indent) {
+        for (int i = 0; i < indent; ++i) {
+            buf.append(' ');
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
new file mode 100644
index 0000000..b757bbb
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public abstract class AbstractGroupingProperty {
+    protected Set<LogicalVariable> columnSet;
+
+    public AbstractGroupingProperty(Set<LogicalVariable> columnSet) {
+        this.columnSet = columnSet;
+    }
+
+    public Set<LogicalVariable> getColumnSet() {
+        return columnSet;
+    }
+
+    public final void normalizeGroupingColumns(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+            List<FunctionalDependency> fds) {
+        replaceGroupingColumnsByEqClasses(equivalenceClasses);
+        applyFDsToGroupingColumns(fds);
+    }
+
+    private void replaceGroupingColumnsByEqClasses(Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
+        if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
+            return;
+        }
+        Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
+        for (LogicalVariable v : columnSet) {
+            EquivalenceClass ec = equivalenceClasses.get(v);
+            if (ec == null) {
+                norm.add(v);
+            } else {
+                if (ec.representativeIsConst()) {
+                    // trivially satisfied, so the var. can be removed
+                } else {
+                    norm.add(ec.getVariableRepresentative());
+                }
+            }
+        }
+        columnSet = norm;
+    }
+
+    private void applyFDsToGroupingColumns(List<FunctionalDependency> fds) {
+        // the set of vars. is unordered
+        // so we try all FDs on all variables (incomplete algo?)
+        if (fds == null || fds.isEmpty()) {
+            return;
+        }
+        Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
+        for (LogicalVariable v : columnSet) {
+            boolean isImpliedByAnFD = false;
+            for (FunctionalDependency fdep : fds) {
+                if (columnSet.containsAll(fdep.getHead()) && fdep.getTail().contains(v)) {
+                    isImpliedByAnFD = true;
+                    norm.addAll(fdep.getHead());
+                    break;
+                }
+
+            }
+            if (!isImpliedByAnFD) {
+                norm.add(v);
+            }
+        }
+        columnSet = norm;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
new file mode 100644
index 0000000..53aea68
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class BroadcastPartitioningProperty implements IPartitioningProperty {
+
+    private INodeDomain domain;
+
+    public BroadcastPartitioningProperty(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+    @Override
+    public PartitioningType getPartitioningType() {
+        return PartitioningType.BROADCAST;
+    }
+
+    @Override
+    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+        // do nothing
+    }
+
+    @Override
+    public void getColumns(Collection<LogicalVariable> columns) {
+    }
+
+    @Override
+    public INodeDomain getNodeDomain() {
+        return domain;
+    }
+
+    @Override
+    public void setNodeDomain(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
new file mode 100644
index 0000000..226f02c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class DefaultNodeGroupDomain implements INodeDomain {
+
+    private String groupName;
+
+    public DefaultNodeGroupDomain(String groupName) {
+        this.groupName = groupName;
+    }
+
+    @Override
+    public boolean sameAs(INodeDomain domain) {
+        return true;
+    }
+
+    @Override
+    public String toString() {
+        return "AsterixDomain(" + groupName + ")";
+    }
+
+    @Override
+    public Integer cardinality() {
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
new file mode 100644
index 0000000..5fbcdb9
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class FileSplitDomain implements INodeDomain {
+
+    private FileSplit[] splits;
+
+    public FileSplitDomain(FileSplit[] splits) {
+        this.splits = splits;
+    }
+
+    @Override
+    public Integer cardinality() {
+        return splits.length;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append("FileSplitDomain[");
+        boolean fst = true;
+        for (FileSplit fs : splits) {
+            if (fst) {
+                fst = false;
+            } else {
+                sb.append(", ");
+            }
+            sb.append(fs.getNodeName() + ":" + fs.getLocalFile());
+        }
+        sb.append(']');
+        return sb.toString();
+    }
+
+    @Override
+    public boolean sameAs(INodeDomain domain) {
+        if (!(domain instanceof FileSplitDomain)) {
+            return false;
+        }
+        FileSplitDomain fsd = (FileSplitDomain) domain;
+        if (fsd.splits.length != splits.length) {
+            return false;
+        }
+        // conservative approach...
+        for (int i = 0; i < splits.length; i++) {
+            if (!ncEq(splits[i], fsd.splits[i])) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private boolean ncEq(FileSplit fs1, FileSplit fs2) {
+        return fs1.getNodeName().equals(fs2.getNodeName());
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
new file mode 100644
index 0000000..822cdc3
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+
+public class FilteredVariablePropagationPolicy extends VariablePropagationPolicy {
+
+    List<LogicalVariable> toPropagate;
+
+    public FilteredVariablePropagationPolicy(List<LogicalVariable> varList) {
+        toPropagate = varList;
+    }
+
+    @Override
+    public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+        for (LogicalVariable v : toPropagate) {
+            target.addVariable(v);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
new file mode 100644
index 0000000..5bf4637
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public final class FunctionalDependency {
+    private List<LogicalVariable> head;
+    private List<LogicalVariable> tail;
+
+    public FunctionalDependency(List<LogicalVariable> head, List<LogicalVariable> tail) {
+        this.head = head;
+        this.tail = tail;
+    }
+
+    public List<LogicalVariable> getHead() {
+        return head;
+    }
+
+    public List<LogicalVariable> getTail() {
+        return tail;
+    }
+
+    @Override
+    public String toString() {
+        return head + "->" + tail;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof FunctionalDependency)) {
+            return false;
+        } else {
+            FunctionalDependency fd = (FunctionalDependency) obj;
+            return fd.getHead().equals(this.head) && fd.getTail().equals(this.tail);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
new file mode 100644
index 0000000..c32fd67
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface ILocalStructuralProperty extends IStructuralProperty {
+    public enum PropertyType {
+        LOCAL_GROUPING_PROPERTY,
+        LOCAL_ORDER_PROPERTY
+    }
+
+    public void getVariables(Collection<LogicalVariable> variables);
+
+    public PropertyType getPropertyType();
+
+    /**
+     * Returns the retained property regarding to a collection of variables,
+     * e.g., some variables used in the property may not exist in the input
+     * collection and hence the data property changes.
+     * 
+     * @param vars
+     *            , an input collection of variables
+     * @return the retained data property.
+     */
+    public ILocalStructuralProperty retainVariables(Collection<LogicalVariable> vars);
+
+    /**
+     * Returns the additional data property within each group, which is dictated by the group keys.
+     * 
+     * @param vars
+     *            , group keys.
+     * @return the additional data property within each group.
+     */
+    public ILocalStructuralProperty regardToGroup(Collection<LogicalVariable> groupKeys);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
new file mode 100644
index 0000000..175f041
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public interface ILogicalPropertiesVector {
+    public Integer getNumberOfTuples();
+
+    public Integer getMaxOutputFrames();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/INodeDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/INodeDomain.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/INodeDomain.java
new file mode 100644
index 0000000..17e97ec
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/INodeDomain.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public interface INodeDomain {
+    public boolean sameAs(INodeDomain domain);
+
+    /** @return the number of nodes in that domain or null if unknown */
+    public Integer cardinality();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
new file mode 100644
index 0000000..bb8cfe2
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IPartitioningProperty extends IStructuralProperty {
+    public enum PartitioningType {
+        UNPARTITIONED, RANDOM, BROADCAST, UNORDERED_PARTITIONED, ORDERED_PARTITIONED
+    }
+
+    static final INodeDomain DOMAIN_FOR_UNPARTITIONED_DATA = new INodeDomain() {
+        @Override
+        public boolean sameAs(INodeDomain domain) {
+            return domain == this;
+        }
+
+        @Override
+        public Integer cardinality() {
+            return null;
+        }
+    };
+
+    public static final IPartitioningProperty UNPARTITIONED = new IPartitioningProperty() {
+
+        @Override
+        public PartitioningType getPartitioningType() {
+            return PartitioningType.UNPARTITIONED;
+        }
+
+        @Override
+        public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+            // do nothing
+        }
+
+        @Override
+        public void getColumns(Collection<LogicalVariable> columns) {
+        }
+
+        @Override
+        public INodeDomain getNodeDomain() {
+            return DOMAIN_FOR_UNPARTITIONED_DATA;
+        }
+
+        @Override
+        public String toString() {
+            return getPartitioningType().toString();
+        }
+
+        @Override
+        public void setNodeDomain(INodeDomain domain) {
+            throw new IllegalStateException();
+        }
+    };
+
+    public abstract PartitioningType getPartitioningType();
+
+    public abstract void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+            List<FunctionalDependency> fds);
+
+    public abstract INodeDomain getNodeDomain();
+
+    public abstract void setNodeDomain(INodeDomain domain);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
new file mode 100644
index 0000000..cd9eb96
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+/**
+ * Implements constraints in between requirements for the children of the same
+ * operator.
+ */
+
+public interface IPartitioningRequirementsCoordinator {
+
+    public static IPartitioningRequirementsCoordinator NO_COORDINATION = new IPartitioningRequirementsCoordinator() {
+
+        @Override
+        public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements,
+                IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context) {
+            return new Pair<Boolean, IPartitioningProperty>(true, requirements);
+        }
+    };
+
+    public static IPartitioningRequirementsCoordinator EQCLASS_PARTITIONING_COORDINATOR = new IPartitioningRequirementsCoordinator() {
+
+        @Override
+        public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty rqdpp,
+                IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context)
+                throws AlgebricksException {
+            if (firstDeliveredPartitioning != null
+                    && firstDeliveredPartitioning.getPartitioningType() == rqdpp.getPartitioningType()) {
+                switch (rqdpp.getPartitioningType()) {
+                    case UNORDERED_PARTITIONED: {
+                        UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
+                        Set<LogicalVariable> set1 = upp1.getColumnSet();
+                        UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) rqdpp;
+                        Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
+                        Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
+                        Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
+
+                        // coordinate from an existing partition property
+                        // (firstDeliveredPartitioning)
+                        for (LogicalVariable v : set1) {
+                            EquivalenceClass ecFirst = eqmap.get(v);
+                            for (LogicalVariable r : uppreq.getColumnSet()) {
+                                EquivalenceClass ec = eqmap.get(r);
+                                if (ecFirst == ec) {
+                                    covered.add(v);
+                                    modifuppreq.add(r);
+                                    break;
+                                }
+                            }
+                        }
+
+                        if (!covered.equals(set1)) {
+                            throw new AlgebricksException("Could not modify " + rqdpp
+                                    + " to agree with partitioning property " + firstDeliveredPartitioning
+                                    + " delivered by previous input operator.");
+                        }
+                        UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq,
+                                rqdpp.getNodeDomain());
+                        return new Pair<Boolean, IPartitioningProperty>(false, upp2);
+                    }
+                    case ORDERED_PARTITIONED: {
+                        throw new NotImplementedException();
+                    }
+                }
+            }
+            return new Pair<Boolean, IPartitioningProperty>(true, rqdpp);
+        }
+
+    };
+
+    public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements,
+            IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
new file mode 100644
index 0000000..45bf1ec
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IPhysicalPropertiesVector {
+
+    public IPartitioningProperty getPartitioningProperty();
+
+    public List<ILocalStructuralProperty> getLocalProperties();
+
+    /**
+     * 
+     * @param reqd
+     *            vector of required properties
+     * @param equivalenceClasses
+     * @param fds
+     * @return a vector of properties from pvector that are not delivered by the
+     *         current vector or null if none
+     */
+    public IPhysicalPropertiesVector getUnsatisfiedPropertiesFrom(IPhysicalPropertiesVector reqd,
+            boolean mayExpandProperties, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+            List<FunctionalDependency> fds);
+
+    public IPhysicalPropertiesVector clone();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
new file mode 100644
index 0000000..716b459
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
+public interface IPropertiesComputer {
+
+    // can return null
+    public IPartitioningProperty computePartitioningProperty(ILogicalExpression expr);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
new file mode 100644
index 0000000..bf8aaa5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IStructuralProperty {
+    public void getColumns(Collection<LogicalVariable> columns);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
new file mode 100644
index 0000000..c267364
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class LocalGroupingProperty extends AbstractGroupingProperty implements ILocalStructuralProperty {
+
+    // preferredOrderEnforcer, if not null, is guaranteed to enforce grouping on
+    // columnSet
+    private List<LogicalVariable> preferredOrderEnforcer;
+
+    public LocalGroupingProperty(Set<LogicalVariable> columnSet) {
+        super(columnSet);
+    }
+
+    public LocalGroupingProperty(Set<LogicalVariable> columnSet, List<LogicalVariable> preferredOrderEnforcer) {
+        this(columnSet);
+        this.preferredOrderEnforcer = preferredOrderEnforcer;
+    }
+
+    @Override
+    public PropertyType getPropertyType() {
+        return PropertyType.LOCAL_GROUPING_PROPERTY;
+    }
+
+    @Override
+    public void getColumns(Collection<LogicalVariable> columns) {
+        columns.addAll(columnSet);
+    }
+
+    @Override
+    public String toString() {
+        return columnSet.toString();
+    }
+
+    @Override
+    public void getVariables(Collection<LogicalVariable> variables) {
+        variables.addAll(columnSet);
+    }
+
+    public List<LogicalVariable> getPreferredOrderEnforcer() {
+        return preferredOrderEnforcer;
+    }
+
+    @Override
+    public ILocalStructuralProperty retainVariables(Collection<LogicalVariable> vars) {
+        Set<LogicalVariable> newVars = new ListSet<LogicalVariable>();
+        newVars.addAll(vars);
+        newVars.retainAll(columnSet);
+        if (columnSet.equals(newVars)) {
+            return new LocalGroupingProperty(columnSet, preferredOrderEnforcer);
+        }
+        // Column set for the retained grouping property
+        Set<LogicalVariable> newColumns = new ListSet<LogicalVariable>();
+        // Matches the prefix of the original column set.
+        for (LogicalVariable v : columnSet) {
+            if (newVars.contains(v)) {
+                newColumns.add(v);
+            } else {
+                break;
+            }
+        }
+        if (newColumns.size() > 0) {
+            return new LocalGroupingProperty(newColumns, preferredOrderEnforcer.subList(0, newColumns.size()));
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public ILocalStructuralProperty regardToGroup(Collection<LogicalVariable> groupKeys) {
+        Set<LogicalVariable> newColumns = new ListSet<LogicalVariable>();
+        for (LogicalVariable v : columnSet) {
+            if (!groupKeys.contains(v)) {
+                newColumns.add(v);
+            }
+        }
+        if (newColumns.size() > 0) {
+            return new LocalGroupingProperty(newColumns, preferredOrderEnforcer.subList(groupKeys.size(),
+                    newColumns.size()));
+        } else {
+            return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
new file mode 100644
index 0000000..6d568f4
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+
+public final class LocalOrderProperty implements ILocalStructuralProperty {
+
+    private List<OrderColumn> orderColumns;
+
+    public LocalOrderProperty(List<OrderColumn> orderColumn) {
+        this.orderColumns = orderColumn;
+    }
+
+    public List<OrderColumn> getOrderColumns() {
+        return orderColumns;
+    }
+
+    public void setOrderColumns(List<OrderColumn> orderColumn) {
+        this.orderColumns = orderColumn;
+    }
+
+    public List<LogicalVariable> getColumns() {
+        List<LogicalVariable> orderVars = new ArrayList<LogicalVariable>();
+        for (OrderColumn oc : orderColumns) {
+            orderVars.add(oc.getColumn());
+        }
+        return orderVars;
+    }
+
+    public List<OrderKind> getOrders() {
+        List<OrderKind> orderKinds = new ArrayList<OrderKind>();
+        for (OrderColumn oc : orderColumns) {
+            orderKinds.add(oc.getOrder());
+        }
+        return orderKinds;
+    }
+
+    @Override
+    public PropertyType getPropertyType() {
+        return PropertyType.LOCAL_ORDER_PROPERTY;
+    }
+
+    @Override
+    public void getColumns(Collection<LogicalVariable> columns) {
+        columns.addAll(getColumns());
+    }
+
+    @Override
+    public String toString() {
+        return orderColumns.toString();
+    }
+
+    @Override
+    public void getVariables(Collection<LogicalVariable> variables) {
+        variables.addAll(getColumns());
+    }
+
+    @Override
+    public boolean equals(Object object) {
+        LocalOrderProperty lop = (LocalOrderProperty) object;
+        return orderColumns.equals(lop.orderColumns);
+    }
+
+    /**
+     * Whether current property implies the required property
+     * 
+     * @param required
+     *            , a required property
+     * @return true if the current property satisfies the required property; otherwise false.
+     */
+    public final boolean implies(ILocalStructuralProperty required) {
+        if (required.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+            return false;
+        }
+        LocalOrderProperty requiredOrderProperty = (LocalOrderProperty) required;
+        Iterator<OrderColumn> requiredColumnIterator = requiredOrderProperty.getOrderColumns().iterator();
+        Iterator<OrderColumn> currentColumnIterator = orderColumns.iterator();
+
+        // Returns true if requiredColumnIterator is a prefix of currentColumnIterator.
+        return isPrefixOf(requiredColumnIterator, currentColumnIterator);
+    }
+
+    private <T> boolean isPrefixOf(Iterator<T> requiredColumnIterator, Iterator<T> currentColumnIterator) {
+        while (requiredColumnIterator.hasNext()) {
+            T oc = requiredColumnIterator.next();
+            if (!currentColumnIterator.hasNext()) {
+                return false;
+            }
+            if (!oc.equals(currentColumnIterator.next())) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public final void normalizeOrderingColumns(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+            List<FunctionalDependency> fds) {
+        replaceOrderingColumnsByEqClasses(equivalenceClasses);
+        applyFDsToOrderingColumns(fds);
+    }
+
+    private void replaceOrderingColumnsByEqClasses(Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
+        if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
+            return;
+        }
+        List<OrderColumn> norm = new ArrayList<OrderColumn>();
+        for (OrderColumn oc : orderColumns) {
+            LogicalVariable v = oc.getColumn();
+            EquivalenceClass ec = equivalenceClasses.get(v);
+            if (ec == null) {
+                norm.add(new OrderColumn(v, oc.getOrder()));
+            } else {
+                if (ec.representativeIsConst()) {
+                    // trivially satisfied, so the var. can be removed
+                } else {
+                    norm.add(new OrderColumn(ec.getVariableRepresentative(), oc.getOrder()));
+                }
+            }
+        }
+        orderColumns = norm;
+    }
+
+    private void applyFDsToOrderingColumns(List<FunctionalDependency> fds) {
+        if (fds == null || fds.isEmpty()) {
+            return;
+        }
+        Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
+        List<LogicalVariable> columns = getColumns();
+        for (LogicalVariable v : columns) {
+            boolean isImpliedByAnFD = false;
+            for (FunctionalDependency fdep : fds) {
+                if (columns.containsAll(fdep.getHead()) && fdep.getTail().contains(v)) {
+                    isImpliedByAnFD = true;
+                    norm.addAll(fdep.getHead());
+                    break;
+                }
+
+            }
+            if (!isImpliedByAnFD) {
+                norm.add(v);
+            }
+        }
+        Set<OrderColumn> impliedColumns = new ListSet<OrderColumn>();
+        for (OrderColumn oc : orderColumns) {
+            if (!norm.contains(oc.getColumn())) {
+                impliedColumns.add(oc);
+            }
+        }
+        orderColumns.removeAll(impliedColumns);
+    }
+
+    @Override
+    public ILocalStructuralProperty retainVariables(Collection<LogicalVariable> vars) {
+        List<LogicalVariable> columns = getColumns();
+        List<LogicalVariable> newVars = new ArrayList<LogicalVariable>();
+        newVars.addAll(vars);
+        newVars.retainAll(columns);
+        List<OrderColumn> newColumns = new ArrayList<OrderColumn>();
+        for (OrderColumn oc : orderColumns) {
+            if (newVars.contains(oc.getColumn())) {
+                newColumns.add(oc);
+            } else {
+                break;
+            }
+        }
+        if (newColumns.size() > 0) {
+            return new LocalOrderProperty(newColumns);
+        } else {
+            return null;
+        }
+    }
+
+    @Override
+    public ILocalStructuralProperty regardToGroup(Collection<LogicalVariable> groupKeys) {
+        List<OrderColumn> newColumns = new ArrayList<OrderColumn>();
+        for (OrderColumn oc : orderColumns) {
+            if (!groupKeys.contains(oc.getColumn())) {
+                newColumns.add(oc);
+            }
+        }
+        if (newColumns.size() > 0) {
+            return new LocalOrderProperty(newColumns);
+        } else {
+            return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
new file mode 100644
index 0000000..848c86e
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class LogicalPropertiesVectorImpl implements ILogicalPropertiesVector {
+
+    private Integer numTuples, maxOutputFrames;
+
+    @Override
+    public String toString() {
+        return "LogicalPropertiesVector [ num.tuples: " + numTuples + ", maxOutputFrames: " + maxOutputFrames + " ]";
+    }
+
+    @Override
+    public Integer getNumberOfTuples() {
+        return numTuples;
+    }
+
+    public void setNumberOfTuples(Integer numTuples) {
+        this.numTuples = numTuples;
+    }
+
+    @Override
+    public Integer getMaxOutputFrames() {
+        return maxOutputFrames;
+    }
+
+    public void setMaxOutputFrames(Integer maxOutputFrames) {
+        this.maxOutputFrames = maxOutputFrames;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderColumn.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderColumn.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderColumn.java
new file mode 100644
index 0000000..a0e76d1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderColumn.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+
+public final class OrderColumn {
+
+    private LogicalVariable column;
+    private OrderKind order;
+
+    public OrderColumn(LogicalVariable column, OrderKind order) {
+        this.column = column;
+        this.order = order;
+    }
+
+    public LogicalVariable getColumn() {
+        return column;
+    }
+
+    public OrderKind getOrder() {
+        return order;
+    }
+
+    public void setColumn(LogicalVariable column) {
+        this.column = column;
+    }
+
+    public void setOrder(OrderKind order) {
+        this.order = order;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (!(obj instanceof OrderColumn)) {
+            return false;
+        } else {
+            OrderColumn oc = (OrderColumn) obj;
+            return column.equals(oc.getColumn()) && order == oc.getOrder();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return column.toString() + "(" + order + ")";
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
new file mode 100644
index 0000000..6f88933
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class OrderedPartitionedProperty implements IPartitioningProperty {
+
+    private ArrayList<OrderColumn> orderColumns;
+    private INodeDomain domain;
+
+    public OrderedPartitionedProperty(ArrayList<OrderColumn> orderColumns, INodeDomain domain) {
+        this.domain = domain;
+        this.orderColumns = orderColumns;
+    }
+
+    public ArrayList<OrderColumn> getOrderColumns() {
+        return orderColumns;
+    }
+
+    public ArrayList<LogicalVariable> getColumns() {
+        ArrayList<LogicalVariable> cols = new ArrayList<LogicalVariable>(orderColumns.size());
+        for (OrderColumn oc : orderColumns) {
+            cols.add(oc.getColumn());
+        }
+        return cols;
+    }
+
+    @Override
+    public PartitioningType getPartitioningType() {
+        return PartitioningType.ORDERED_PARTITIONED;
+    }
+
+    @Override
+    public String toString() {
+        return getPartitioningType().toString() + orderColumns;
+    }
+
+    @Override
+    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+        orderColumns = PropertiesUtil.replaceOrderColumnsByEqClasses(orderColumns, equivalenceClasses);
+        orderColumns = PropertiesUtil.applyFDsToOrderColumns(orderColumns, fds);
+    }
+
+    @Override
+    public void getColumns(Collection<LogicalVariable> columns) {
+        for (OrderColumn oc : orderColumns) {
+            columns.add(oc.getColumn());
+        }
+    }
+
+    @Override
+    public INodeDomain getNodeDomain() {
+        return domain;
+    }
+
+    @Override
+    public void setNodeDomain(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
new file mode 100644
index 0000000..ce0907f
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class PhysicalRequirements {
+    private final IPhysicalPropertiesVector[] requiredProperties;
+    private final IPartitioningRequirementsCoordinator partitioningCoordinator;
+
+    public PhysicalRequirements(IPhysicalPropertiesVector[] requiredProperties,
+            IPartitioningRequirementsCoordinator partitioningCoordinator) {
+        this.requiredProperties = requiredProperties;
+        this.partitioningCoordinator = partitioningCoordinator;
+    }
+
+    public IPhysicalPropertiesVector[] getRequiredProperties() {
+        return requiredProperties;
+    }
+
+    public IPartitioningRequirementsCoordinator getPartitioningCoordinator() {
+        return partitioningCoordinator;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
new file mode 100644
index 0000000..b651e59
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
@@ -0,0 +1,320 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+
+public class PropertiesUtil {
+
+    public Set<LogicalVariable> closureUnderFDs(Collection<LogicalVariable> vars, List<FunctionalDependency> fdList) {
+        Set<LogicalVariable> k = new ListSet<LogicalVariable>(vars);
+        boolean change;
+        do {
+            change = false;
+            for (FunctionalDependency fd : fdList) {
+                List<LogicalVariable> h = fd.getHead();
+                if (k.containsAll(h)) {
+                    List<LogicalVariable> t = fd.getTail();
+                    for (LogicalVariable v : t) {
+                        if (!(k.contains(v))) {
+                            k.add(v);
+                            change = true;
+                        }
+                    }
+                }
+            }
+        } while (change);
+        return k;
+    }
+
+    public static boolean matchLocalProperties(List<ILocalStructuralProperty> reqd,
+            List<ILocalStructuralProperty> dlvd, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+            List<FunctionalDependency> fds) {
+        if (reqd == null) {
+            return true;
+        }
+        if (dlvd == null) {
+            return false;
+        }
+        normalizeLocals(reqd, equivalenceClasses, fds);
+        normalizeLocals(dlvd, equivalenceClasses, fds);
+
+        ListIterator<ILocalStructuralProperty> dlvdIter = dlvd.listIterator();
+
+        Set<LogicalVariable> rqdCols = new ListSet<LogicalVariable>();
+        Set<LogicalVariable> dlvdCols = new ListSet<LogicalVariable>();
+        for (ILocalStructuralProperty r : reqd) {
+            if (r.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
+                rqdCols.clear();
+                r.getVariables(rqdCols);
+            }
+            boolean implied = false;
+            while (!implied && dlvdIter.hasNext()) {
+                ILocalStructuralProperty d = dlvdIter.next();
+                switch (r.getPropertyType()) {
+                    case LOCAL_ORDER_PROPERTY: {
+                        if (d.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+                            return false;
+                        }
+                        LocalOrderProperty lop = (LocalOrderProperty) d;
+                        if (lop.implies(r)) {
+                            implied = true;
+                        } else {
+                            return false;
+                        }
+                        break;
+                    }
+                    case LOCAL_GROUPING_PROPERTY: {
+                        dlvdCols.clear();
+                        d.getColumns(dlvdCols);
+                        if (d.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+                            implied = isPrefixOf(rqdCols.iterator(), dlvdCols.iterator());
+                        } else {
+                            implied = rqdCols.equals(dlvdCols) || isPrefixOf(rqdCols.iterator(), dlvdCols.iterator());
+                        }
+                        break;
+                    }
+                    default: {
+                        throw new IllegalStateException();
+                    }
+                }
+            }
+            if (!implied) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public static boolean matchPartitioningProps(IPartitioningProperty reqd, IPartitioningProperty dlvd,
+            boolean mayExpandProperties) {
+        INodeDomain dom1 = reqd.getNodeDomain();
+        INodeDomain dom2 = dlvd.getNodeDomain();
+        if (dom1 != null && dom2 != null && !dom1.sameAs(dom2)) {
+            return false;
+        }
+
+        switch (reqd.getPartitioningType()) {
+            case RANDOM: {
+                // anything matches RANDOM
+                return true;
+            }
+            case UNORDERED_PARTITIONED: {
+                switch (dlvd.getPartitioningType()) {
+                    case UNORDERED_PARTITIONED: {
+                        UnorderedPartitionedProperty ur = (UnorderedPartitionedProperty) reqd;
+                        UnorderedPartitionedProperty ud = (UnorderedPartitionedProperty) dlvd;
+                        if (mayExpandProperties) {
+                            return isPrefixOf(ud.getColumnSet().iterator(), ur.getColumnSet().iterator());
+                        } else {
+                            return ur.getColumnSet().equals(ud.getColumnSet());
+                        }
+                    }
+                    case ORDERED_PARTITIONED: {
+                        UnorderedPartitionedProperty ur = (UnorderedPartitionedProperty) reqd;
+                        OrderedPartitionedProperty od = (OrderedPartitionedProperty) dlvd;
+                        if (mayExpandProperties) {
+                            List<LogicalVariable> dlvdSortColumns = orderColumnsToVariables(od.getOrderColumns());
+                            return isPrefixOf(dlvdSortColumns.iterator(), ur.getColumnSet().iterator());
+                        } else {
+                            return ur.getColumnSet().containsAll(od.getOrderColumns())
+                                    && od.getOrderColumns().containsAll(ur.getColumnSet());
+                        }
+                    }
+                    default: {
+                        return false;
+                    }
+                }
+            }
+            case ORDERED_PARTITIONED: {
+                switch (dlvd.getPartitioningType()) {
+                    case ORDERED_PARTITIONED: {
+                        OrderedPartitionedProperty or = (OrderedPartitionedProperty) reqd;
+                        OrderedPartitionedProperty od = (OrderedPartitionedProperty) dlvd;
+                        if (mayExpandProperties) {
+                            return isPrefixOf(od.getOrderColumns().iterator(), or.getOrderColumns().iterator());
+                        } else {
+                            return od.getOrderColumns().equals(or.getOrderColumns());
+                        }
+                    }
+                    default: {
+                        return false;
+                    }
+                }
+            }
+            default: {
+                return (dlvd.getPartitioningType() == reqd.getPartitioningType());
+            }
+        }
+    }
+
+    /**
+     * Converts a list of OrderColumns to a list of LogicalVariables.
+     * 
+     * @param orderColumns
+     *            , a list of OrderColumns
+     * @return the list of LogicalVariables
+     */
+    private static List<LogicalVariable> orderColumnsToVariables(List<OrderColumn> orderColumns) {
+        List<LogicalVariable> columns = new ArrayList<LogicalVariable>();
+        for (OrderColumn oc : orderColumns) {
+            columns.add(oc.getColumn());
+        }
+        return columns;
+    }
+
+    /**
+     * @param pref
+     * @param target
+     * @return true iff pref is a prefix of target
+     */
+    private static <T> boolean isPrefixOf(Iterator<T> pref, Iterator<T> target) {
+        while (pref.hasNext()) {
+            T v = pref.next();
+            if (!target.hasNext()) {
+                return false;
+            }
+            if (!v.equals(target.next())) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public static ArrayList<OrderColumn> applyFDsToOrderColumns(ArrayList<OrderColumn> orderColumns,
+            List<FunctionalDependency> fds) {
+        // the set of vars. is ordered
+        // so we try the variables in order from last to first
+        if (fds == null || fds.isEmpty()) {
+            return orderColumns;
+        }
+
+        int deleted = 0;
+        for (int i = orderColumns.size() - 1; i >= 0; i--) {
+            for (FunctionalDependency fdep : fds) {
+                if (impliedByPrefix(orderColumns, i, fdep)) {
+                    orderColumns.set(i, null);
+                    deleted++;
+                    break;
+                }
+            }
+        }
+        ArrayList<OrderColumn> norm = new ArrayList<OrderColumn>(orderColumns.size() - deleted);
+        for (OrderColumn oc : orderColumns) {
+            if (oc != null) {
+                norm.add(oc);
+            }
+        }
+        return norm;
+    }
+
+    public static ArrayList<OrderColumn> replaceOrderColumnsByEqClasses(ArrayList<OrderColumn> orderColumns,
+            Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
+        if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
+            return orderColumns;
+        }
+        ArrayList<OrderColumn> norm = new ArrayList<OrderColumn>();
+        for (OrderColumn v : orderColumns) {
+            EquivalenceClass ec = equivalenceClasses.get(v.getColumn());
+            if (ec == null) {
+                norm.add(v);
+            } else {
+                if (ec.representativeIsConst()) {
+                    // trivially satisfied, so the var. can be removed
+                } else {
+                    norm.add(new OrderColumn(ec.getVariableRepresentative(), v.getOrder()));
+                }
+            }
+        }
+        return norm;
+    }
+
+    private static boolean impliedByPrefix(ArrayList<OrderColumn> vars, int i, FunctionalDependency fdep) {
+        if (!fdep.getTail().contains(vars.get(i).getColumn())) {
+            return false;
+        }
+        boolean fdSat = true;
+        for (LogicalVariable pv : fdep.getHead()) {
+            boolean isInPrefix = false;
+            for (int j = 0; j < i; j++) {
+                if (vars.get(j).getColumn().equals(pv)) {
+                    isInPrefix = true;
+                    break;
+                }
+            }
+            if (!isInPrefix) {
+                fdSat = false;
+                break;
+            }
+        }
+        return fdSat;
+    }
+
+    private static void normalizeLocals(List<ILocalStructuralProperty> props,
+            Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+        ListIterator<ILocalStructuralProperty> propIter = props.listIterator();
+        int pos = -1;
+        while (propIter.hasNext()) {
+            ILocalStructuralProperty p = propIter.next();
+            if (p.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
+                ((LocalGroupingProperty) p).normalizeGroupingColumns(equivalenceClasses, fds);
+                pos++;
+            } else {
+                ((LocalOrderProperty) p).normalizeOrderingColumns(equivalenceClasses, fds);
+                pos++;
+            }
+        }
+
+        if (pos < 1) {
+            return;
+        }
+
+        while (propIter.hasPrevious()) {
+            ILocalStructuralProperty p = propIter.previous();
+            ListIterator<ILocalStructuralProperty> secondIter = props.listIterator(pos);
+            pos--;
+            Set<LogicalVariable> cols = new ListSet<LogicalVariable>();
+            while (secondIter.hasPrevious()) {
+                secondIter.previous().getColumns(cols);
+            }
+            secondIter = null;
+            for (FunctionalDependency fdep : fds) {
+                LinkedList<LogicalVariable> columnsOfP = new LinkedList<LogicalVariable>();
+                p.getColumns(columnsOfP);
+                if (impliedByPrefix(columnsOfP, cols, fdep)) {
+                    propIter.remove();
+                    break;
+                }
+            }
+        }
+    }
+
+    private static boolean impliedByPrefix(List<LogicalVariable> colsOfProp, Set<LogicalVariable> colsOfPrefix,
+            FunctionalDependency fdep) {
+        return fdep.getTail().containsAll(colsOfProp) && colsOfPrefix.containsAll(fdep.getHead());
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
new file mode 100644
index 0000000..06008a6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class RandomPartitioningProperty implements IPartitioningProperty {
+
+    private INodeDomain domain;
+
+    public RandomPartitioningProperty(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+    @Override
+    public PartitioningType getPartitioningType() {
+        return PartitioningType.RANDOM;
+    }
+
+    @Override
+    public String toString() {
+        return getPartitioningType() + " domain:" + domain;
+    }
+
+    @Override
+    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+        // do nothing
+    }
+
+    @Override
+    public void getColumns(Collection<LogicalVariable> columns) {
+    }
+
+    @Override
+    public INodeDomain getNodeDomain() {
+        return domain;
+    }
+
+    @Override
+    public void setNodeDomain(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
new file mode 100644
index 0000000..688523b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class ResultSetDomain implements INodeDomain {
+    @Override
+    public boolean sameAs(INodeDomain domain) {
+        return true;
+    }
+
+    @Override
+    public Integer cardinality() {
+        return 0;
+    }
+}

[49/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java
new file mode 100644
index 0000000..436be40
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public interface ICompiler {
+    public void optimize() throws AlgebricksException;
+
+    public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompilerFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompilerFactory.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompilerFactory.java
new file mode 100644
index 0000000..66f3b81
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompilerFactory.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface ICompilerFactory {
+    ICompiler createCompiler(ILogicalPlan plan, IMetadataProvider<?, ?> metadata, int varCounter);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
new file mode 100644
index 0000000..2062c5f
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * 
+ * Runs each rule until it produces no changes. Then the whole collection of
+ * rules is run again until no change is made.
+ * 
+ * 
+ * @author Nicola
+ * 
+ */
+
+public class PrioritizedRuleController extends AbstractRuleController {
+
+    public PrioritizedRuleController() {
+        super();
+    }
+
+    @Override
+    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
+            throws AlgebricksException {
+        boolean anyRuleFired = false;
+        boolean anyChange = false;
+        do {
+            anyChange = false;
+            for (IAlgebraicRewriteRule r : rules) {
+                while (true) {
+                    boolean ruleFired = rewriteOperatorRef(root, r);
+                    if (ruleFired) {
+                        anyChange = true;
+                        anyRuleFired = true;
+                    } else {
+                        break; // go to next rule
+                    }
+                }
+            }
+        } while (anyChange);
+        return anyRuleFired;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
new file mode 100644
index 0000000..2f65c8a
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Runs rules sequentially (round-robin), until one iteration over all rules
+ * produces no change.
+ * 
+ * @author Nicola
+ */
+public class SequentialFixpointRuleController extends AbstractRuleController {
+
+    private boolean fullDfs;
+
+    public SequentialFixpointRuleController(boolean fullDfs) {
+        super();
+        this.fullDfs = fullDfs;
+    }
+
+    @Override
+    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
+            Collection<IAlgebraicRewriteRule> ruleCollection) throws AlgebricksException {
+        boolean anyRuleFired = false;
+        boolean anyChange = false;
+        do {
+            anyChange = false;
+            for (IAlgebraicRewriteRule rule : ruleCollection) {
+                boolean ruleFired = rewriteOperatorRef(root, rule, true, fullDfs);
+                if (ruleFired) {
+                    anyChange = true;
+                    anyRuleFired = true;
+                }
+            }
+        } while (anyChange);
+        return anyRuleFired;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
new file mode 100644
index 0000000..0f8ec80
--- /dev/null
+++ b/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class SequentialOnceRuleController extends AbstractRuleController {
+
+    private final boolean enterNestedPlans;
+
+    public SequentialOnceRuleController(boolean enterNestedPlans) {
+        super();
+        this.enterNestedPlans = enterNestedPlans;
+    }
+
+    @Override
+    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
+            throws AlgebricksException {
+        boolean fired = false;
+        for (IAlgebraicRewriteRule rule : rules) {
+            if (rewriteOperatorRef(root, rule, enterNestedPlans, true)) {
+                fired = true;
+            }
+        }
+        return fired;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java
deleted file mode 100644
index 3dd4704..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public class Counter {
-    private int counter = 0;
-
-    public Counter() {
-    }
-
-    public Counter(int initial) {
-        counter = initial;
-    }
-
-    public int get() {
-        return counter;
-    }
-
-    public void inc() {
-        ++counter;
-    }
-
-    public void set(int newStart) {
-        counter = newStart;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
deleted file mode 100644
index ab95c9c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-
-public final class EquivalenceClass {
-    private Set<ILogicalExpression> expressionMembers = new ListSet<ILogicalExpression>();
-    private Set<LogicalVariable> members = new ListSet<LogicalVariable>();
-    private ILogicalExpression constRepresentative;
-    private LogicalVariable variableRepresentative;
-    private boolean representativeIsConst;
-
-    public EquivalenceClass(Collection<LogicalVariable> members, ILogicalExpression constRepresentative) {
-        this.members.addAll(members);
-        this.constRepresentative = constRepresentative;
-        representativeIsConst = true;
-    }
-
-    public EquivalenceClass(Collection<LogicalVariable> members, LogicalVariable variableRepresentative) {
-        this.members.addAll(members);
-        this.variableRepresentative = variableRepresentative;
-        representativeIsConst = false;
-    }
-
-    public EquivalenceClass(Collection<LogicalVariable> members, ILogicalExpression constRepresentative,
-            Collection<ILogicalExpression> expressionMembers) {
-        this(members, constRepresentative);
-        this.expressionMembers.addAll(expressionMembers);
-    }
-
-    public EquivalenceClass(Collection<LogicalVariable> members, LogicalVariable variableRepresentative,
-            Collection<ILogicalExpression> expressionMembers) {
-        this(members, variableRepresentative);
-        this.expressionMembers.addAll(expressionMembers);
-    }
-
-    public boolean representativeIsConst() {
-        return representativeIsConst;
-    }
-
-    public Collection<LogicalVariable> getMembers() {
-        return members;
-    }
-
-    public Collection<ILogicalExpression> getExpressionMembers() {
-        return expressionMembers;
-    }
-
-    public boolean contains(LogicalVariable var) {
-        return members.contains(var);
-    }
-
-    public boolean contains(ILogicalExpression expr) {
-        return expressionMembers.contains(expr);
-    }
-
-    public ILogicalExpression getConstRepresentative() {
-        return constRepresentative;
-    }
-
-    public LogicalVariable getVariableRepresentative() {
-        return variableRepresentative;
-    }
-
-    public void setConstRepresentative(ILogicalExpression constRepresentative) {
-        this.constRepresentative = constRepresentative;
-        this.representativeIsConst = true;
-    }
-
-    public void setVariableRepresentative(LogicalVariable variableRepresentative) {
-        this.variableRepresentative = variableRepresentative;
-        this.representativeIsConst = false;
-    }
-
-    public void merge(EquivalenceClass ec2) {
-        members.addAll(ec2.getMembers());
-        if (!representativeIsConst && ec2.representativeIsConst()) {
-            representativeIsConst = true;
-            constRepresentative = ec2.getConstRepresentative();
-        }
-        expressionMembers.addAll(ec2.getExpressionMembers());
-    }
-
-    public void addMember(LogicalVariable v) {
-        members.add(v);
-    }
-
-    public EquivalenceClass cloneEquivalenceClass() {
-        List<LogicalVariable> membersClone = new LinkedList<LogicalVariable>();
-        membersClone.addAll(members);
-        EquivalenceClass ec;
-        if (representativeIsConst()) {
-            ec = new EquivalenceClass(membersClone, constRepresentative);
-        } else {
-            ec = new EquivalenceClass(membersClone, variableRepresentative);
-        }
-        return ec;
-    }
-
-    @Override
-    public String toString() {
-        return "(<" + (representativeIsConst ? constRepresentative : variableRepresentative) + "> " + members + ";"
-                + expressionMembers + ")";
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof EquivalenceClass)) {
-            return false;
-        } else {
-            EquivalenceClass ec = (EquivalenceClass) obj;
-            if (!members.equals(ec.getMembers())) {
-                return false;
-            }
-            if (!expressionMembers.equals(ec.getExpressionMembers())) {
-                return false;
-            }
-            if (representativeIsConst) {
-                return ec.representativeIsConst() && (constRepresentative.equals(ec.getConstRepresentative()));
-            } else {
-                return !ec.representativeIsConst() && (variableRepresentative.equals(ec.getVariableRepresentative()));
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
deleted file mode 100644
index 4e341e4..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public interface IHyracksJobBuilder {
-    public enum TargetConstraint {
-        ONE,
-        SAME_COUNT
-    }
-
-    public void contributeHyracksOperator(ILogicalOperator op, IOperatorDescriptor opDesc);
-
-    public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apc);
-
-    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc);
-
-    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc,
-            AlgebricksPartitionConstraint pc);
-
-    /**
-     * inputs are numbered starting from 0
-     */
-    public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex);
-
-    public void contributeConnector(ILogicalOperator exchgOp, IConnectorDescriptor conn);
-
-    public void contributeConnectorWithTargetConstraint(ILogicalOperator exchgOp, IConnectorDescriptor conn,
-            TargetConstraint numberOfTargetPartitions);
-
-    public JobSpecification getJobSpec();
-
-    /**
-     * to be called only after all the graph information is added
-     */
-    public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
deleted file mode 100644
index c4ae9bc..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-public interface ILogicalExpression {
-
-    public LogicalExpressionTag getExpressionTag();
-
-    public abstract <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException;
-
-    public void getUsedVariables(Collection<LogicalVariable> vars);
-
-    public void substituteVar(LogicalVariable v1, LogicalVariable v2);
-
-    // constraints (e.g., FDs, equivalences)
-
-    /**
-     * @param fds
-     *            Output argument: functional dependencies that can be inferred
-     *            from this expression.
-     * @param equivClasses
-     *            Output argument: Equivalence classes that can be inferred from
-     *            this expression.
-     */
-    public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
-            Map<LogicalVariable, EquivalenceClass> equivClasses);
-
-    /**
-     * @param fds
-     *            Output argument: functional dependencies that can be inferred
-     *            from this expression.
-     * @param outerVars
-     *            Input argument: variables coming from outer branch(es), e.g.,
-     *            the left branch of a left outer join.
-     */
-    public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars);
-
-    /**
-     * @param conjs
-     *            Output argument: a list of expression whose conjunction, in
-     *            any order, can replace the current expression.
-     * @return true if the expression can be broken in at least two conjuncts,
-     *         false otherwise.
-     */
-    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs);
-
-    public abstract ILogicalExpression cloneExpression();
-
-    public boolean isFunctional();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
deleted file mode 100644
index d91cac0..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-
-public interface ILogicalOperator {
-
-    public LogicalOperatorTag getOperatorTag();
-
-    public ExecutionMode getExecutionMode();
-
-    public List<Mutable<ILogicalOperator>> getInputs();
-
-    boolean hasInputs();
-
-    public void recomputeSchema() throws AlgebricksException;
-
-    public List<LogicalVariable> getSchema();
-
-    /*
-     * 
-     * support for visitors
-     */
-
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException;
-
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException;
-
-    public boolean isMap();
-
-    public Map<String, Object> getAnnotations();
-
-    public void removeAnnotation(String annotationName);
-
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException;
-
-    // variables
-
-    /**
-     * Get the variable propogation policy from this operator's input to its
-     * output.
-     * 
-     * @return The VariablePropogationPolicy.
-     */
-    public VariablePropagationPolicy getVariablePropagationPolicy();
-
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException;
-
-    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException;
-
-    // structural properties
-
-    /**
-     * @return for each child, one vector of required physical properties
-     */
-
-    public PhysicalRequirements getRequiredPhysicalPropertiesForChildren(IPhysicalPropertiesVector requiredProperties);
-
-    /**
-     * @return the physical properties that this operator delivers, based on
-     *         what its children deliver
-     */
-
-    public IPhysicalPropertiesVector getDeliveredPhysicalProperties();
-
-    public void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException;
-
-    /**
-     * Indicates whether the expressions used by this operator must be variable reference expressions.
-     */
-    public boolean requiresVariableReferenceExpressions();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
deleted file mode 100644
index 6db8f8d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-public interface ILogicalPlan {
-    public List<Mutable<ILogicalOperator>> getRoots();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
deleted file mode 100644
index 06f619b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public interface ILogicalPlanAndMetadata {
-    public ILogicalPlan getPlan();
-
-    public IMetadataProvider<?, ?> getMetadataProvider();
-
-    public AlgebricksPartitionConstraint getClusterLocations();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
deleted file mode 100644
index 9b9273a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-
-public interface IOptimizationContext extends ITypingContext {
-
-    public abstract int getVarCounter();
-
-    public abstract void setVarCounter(int varCounter);
-
-    public abstract LogicalVariable newVar();
-
-    public abstract IMetadataProvider<?, ?> getMetadataProvider();
-
-    public abstract void setMetadataDeclarations(IMetadataProvider<?, ?> metadataProvider);
-
-    public abstract boolean checkIfInDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op);
-
-    public abstract void addToDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op);
-
-    /*
-     * returns true if op1 and op2 have already been compared
-     */
-    public abstract boolean checkAndAddToAlreadyCompared(ILogicalOperator op1, ILogicalOperator op2);
-    
-    public abstract void removeFromAlreadyCompared(ILogicalOperator op1);
-
-    public abstract void addNotToBeInlinedVar(LogicalVariable var);
-
-    public abstract boolean shouldNotBeInlined(LogicalVariable var);
-
-    public abstract void addPrimaryKey(FunctionalDependency pk);
-
-    public abstract List<LogicalVariable> findPrimaryKey(LogicalVariable recordVar);
-
-    public abstract void putEquivalenceClassMap(ILogicalOperator op, Map<LogicalVariable, EquivalenceClass> eqClassMap);
-
-    public abstract Map<LogicalVariable, EquivalenceClass> getEquivalenceClassMap(ILogicalOperator op);
-
-    public abstract void putFDList(ILogicalOperator op, List<FunctionalDependency> fdList);
-
-    public abstract List<FunctionalDependency> getFDList(ILogicalOperator op);
-
-    public abstract void putLogicalPropertiesVector(ILogicalOperator op, ILogicalPropertiesVector v);
-
-    public abstract ILogicalPropertiesVector getLogicalPropertiesVector(ILogicalOperator op);
-
-    public abstract IExpressionEvalSizeComputer getExpressionEvalSizeComputer();
-
-    public abstract IVariableEvalSizeEnvironment getVariableEvalSizeEnvironment();
-
-    public abstract IMergeAggregationExpressionFactory getMergeAggregationExpressionFactory();
-
-    public abstract PhysicalOptimizationConfig getPhysicalOptimizationConfig();
-
-    public abstract void invalidateTypeEnvironmentForOperator(ILogicalOperator op);
-
-    public abstract void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException;
-
-    public abstract void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars);
-    
-    public abstract LogicalOperatorPrettyPrintVisitor getPrettyPrintVisitor();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
deleted file mode 100644
index dafe300..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-
-public interface IPhysicalOperator {
-
-    public PhysicalOperatorTag getOperatorTag();
-
-    /**
-     * @param op
-     *            the logical operator this physical operator annotates
-     * @param reqdByParent
-     *            parent's requirements, which are not enforced for now, as we
-     *            only explore one plan
-     * @return for each child, one vector of required physical properties
-     */
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent);
-
-    /**
-     * @return the physical properties that this operator delivers, based on
-     *         what its children deliver
-     */
-    public IPhysicalPropertiesVector getDeliveredProperties();
-
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException;
-
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException;
-
-    public void disableJobGenBelowMe();
-
-    public boolean isJobGenDisabledBelowMe();
-
-    public boolean isMicroOperator();
-
-    public void setHostQueryContext(Object context);
-
-    public Object getHostQueryContext();
-
-    /**
-     * @return labels (0 or 1) for each input and output indicating the dependency between them.
-     *         The edges labeled as 1 must wait for the edges with label 0.
-     */
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op);
-
-    /*
-     * This is needed to have a kind of cost based decision on whether to merge the shared subplans and materialize the result.
-     * If the subgraph whose result we would like to materialize has an operator that is computationally expensive, we assume
-     * it is cheaper to materialize the result of this subgraph and read from the file rather than recomputing it.
-     */
-    public boolean expensiveThanMaterialization();
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
deleted file mode 100644
index 83bc702..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public enum LogicalExpressionTag {
-    FUNCTION_CALL, VARIABLE, CONSTANT
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
deleted file mode 100644
index 9ddd927..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public enum LogicalOperatorTag {
-    AGGREGATE,
-    ASSIGN,
-    DATASOURCESCAN,
-    DISTINCT,
-    DISTRIBUTE_RESULT,
-    EMPTYTUPLESOURCE,
-    EXCHANGE,
-    EXTENSION_OPERATOR,
-    EXTERNAL_LOOKUP,
-    GROUP,
-    INDEX_INSERT_DELETE,
-    INNERJOIN,
-    INSERT_DELETE,
-    LEFTOUTERJOIN,
-    LIMIT,
-    MATERIALIZE,
-    NESTEDTUPLESOURCE,
-    ORDER,
-    PARTITIONINGSPLIT,
-    PROJECT,
-    REPLICATE,
-    RUNNINGAGGREGATE,
-    SCRIPT,
-    SELECT,
-    SINK,
-    SUBPLAN,
-    TOKENIZE,
-    UNIONALL,
-    UNNEST,
-    UNNEST_MAP,
-    UPDATE,
-    WRITE,
-    WRITE_RESULT,
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java
deleted file mode 100644
index 07596b1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-/**
- * Represents a logical variable in an asterix logical plan.
- * 
- * @author Vinayak R. Borkar [vborkar@ics.uci.edu]
- */
-public final class LogicalVariable {
-    private final int id;
-
-    public LogicalVariable(int id) {
-        this.id = id;
-    }
-
-    public int getId() {
-        return id;
-    }
-
-    @Override
-    public String toString() {
-        return "$$" + id;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof LogicalVariable)) {
-            return false;
-        } else {
-            return id == ((LogicalVariable) obj).getId();
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        return id;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
deleted file mode 100644
index 251edca..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public interface OperatorAnnotations {
-    // hints
-    public static final String USE_HASH_GROUP_BY = "USE_HASH_GROUP_BY"; // -->
-    public static final String USE_EXTERNAL_GROUP_BY = "USE_EXTERNAL_GROUP_BY"; // -->
-    public static final String USE_RANGE_CONNECTOR = "USE_RANGE_CONNECTOR"; // -->
-    // Boolean
-    public static final String CARDINALITY = "CARDINALITY"; // -->
-    // Integer
-    public static final String MAX_NUMBER_FRAMES = "MAX_NUMBER_FRAMES"; // -->
-    // Integer
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
deleted file mode 100644
index 0c9e89a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public enum PhysicalOperatorTag {
-    AGGREGATE,
-    ASSIGN,
-    BROADCAST_EXCHANGE,
-    BTREE_SEARCH,
-    BULKLOAD,
-    DATASOURCE_SCAN,
-    DISTRIBUTE_RESULT,
-    EMPTY_TUPLE_SOURCE,
-    EXTENSION_OPERATOR,
-    EXTERNAL_GROUP_BY,
-    EXTERNAL_LOOKUP,
-    HASH_GROUP_BY,
-    HASH_PARTITION_EXCHANGE,
-    HASH_PARTITION_MERGE_EXCHANGE,
-    HDFS_READER,
-    HYBRID_HASH_JOIN,
-    IN_MEMORY_HASH_JOIN,
-    IN_MEMORY_STABLE_SORT,
-    INDEX_BULKLOAD,
-    INDEX_INSERT_DELETE,
-    INSERT_DELETE,
-    LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH,
-    MATERIALIZE,
-    MICRO_PRE_CLUSTERED_GROUP_BY,
-    NESTED_LOOP,
-    NESTED_TUPLE_SOURCE,
-    ONE_TO_ONE_EXCHANGE,
-    PARTITIONINGSPLIT,
-    PRE_CLUSTERED_GROUP_BY,
-    PRE_SORTED_DISTINCT_BY,
-    RANDOM_PARTITION_EXCHANGE,
-    RANDOM_MERGE_EXCHANGE,
-    RANGE_PARTITION_EXCHANGE,
-    RANGE_PARTITION_MERGE_EXCHANGE,
-    RTREE_SEARCH,
-    RUNNING_AGGREGATE,
-    SINGLE_PARTITION_INVERTED_INDEX_SEARCH,
-    SINK,
-    SINK_WRITE,
-    SORT_GROUP_BY,
-    SORT_MERGE_EXCHANGE,
-    SPLIT,
-    STABLE_SORT,
-    STATS,
-    STREAM_LIMIT,
-    STREAM_PROJECT,
-    STREAM_SELECT,
-    STRING_STREAM_SCRIPT,
-    SUBPLAN,
-    TOKENIZE,
-    UNION_ALL,
-    UNNEST,
-    UPDATE,
-    WRITE_RESULT,
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
deleted file mode 100644
index 1792641..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
+++ /dev/null
@@ -1,355 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-
-public abstract class AbstractFunctionCallExpression extends AbstractLogicalExpression {
-
-    public enum FunctionKind {
-        SCALAR,
-        STATEFUL,
-        AGGREGATE,
-        UNNEST
-    }
-
-    protected IFunctionInfo finfo;
-    final private List<Mutable<ILogicalExpression>> arguments;
-    private Object[] opaqueParameters;
-    private final FunctionKind kind;
-    private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
-
-    public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
-            List<Mutable<ILogicalExpression>> arguments) {
-        this.kind = kind;
-        this.finfo = finfo;
-        this.arguments = arguments;
-    }
-
-    public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo) {
-        this.kind = kind;
-        this.finfo = finfo;
-        this.arguments = new ArrayList<Mutable<ILogicalExpression>>();
-    }
-
-    public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
-            Mutable<ILogicalExpression>... expressions) {
-        this(kind, finfo);
-        for (Mutable<ILogicalExpression> e : expressions) {
-            this.arguments.add(e);
-        }
-    }
-
-    public void setOpaqueParameters(Object[] opaqueParameters) {
-        this.opaqueParameters = opaqueParameters;
-    }
-
-    public Object[] getOpaqueParameters() {
-        return opaqueParameters;
-    }
-
-    public FunctionKind getKind() {
-        return kind;
-    }
-
-    protected List<Mutable<ILogicalExpression>> cloneArguments() {
-        List<Mutable<ILogicalExpression>> clonedArgs = new ArrayList<Mutable<ILogicalExpression>>(arguments.size());
-        for (Mutable<ILogicalExpression> e : arguments) {
-            ILogicalExpression e2 = ((AbstractLogicalExpression) e.getValue()).cloneExpression();
-            clonedArgs.add(new MutableObject<ILogicalExpression>(e2));
-        }
-        return clonedArgs;
-    }
-
-    public FunctionIdentifier getFunctionIdentifier() {
-        return finfo.getFunctionIdentifier();
-    }
-
-    public IFunctionInfo getFunctionInfo() {
-        return finfo;
-    }
-
-    public void setFunctionInfo(IFunctionInfo finfo) {
-        this.finfo = finfo;
-    }
-
-    public List<Mutable<ILogicalExpression>> getArguments() {
-        return arguments;
-    }
-
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("function-call: " + finfo.getFunctionIdentifier() + ", Args:[");
-        // + arguments;
-        boolean first = true;
-        for (Mutable<ILogicalExpression> ref : arguments) {
-            if (first) {
-                first = false;
-            } else {
-                sb.append(", ");
-            }
-            sb.append(ref.getValue());
-        }
-        sb.append("]");
-        return sb.toString();
-    }
-
-    @Override
-    public LogicalExpressionTag getExpressionTag() {
-        return LogicalExpressionTag.FUNCTION_CALL;
-    }
-
-    @Override
-    public void getUsedVariables(Collection<LogicalVariable> vars) {
-        for (Mutable<ILogicalExpression> arg : arguments) {
-            arg.getValue().getUsedVariables(vars);
-        }
-    }
-
-    @Override
-    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
-        for (Mutable<ILogicalExpression> arg : arguments) {
-            arg.getValue().substituteVar(v1, v2);
-        }
-    }
-
-    @Override
-    public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
-            Map<LogicalVariable, EquivalenceClass> equivClasses) {
-        FunctionIdentifier funId = getFunctionIdentifier();
-        if (funId.equals(AlgebricksBuiltinFunctions.AND)) {
-            for (Mutable<ILogicalExpression> a : arguments) {
-                a.getValue().getConstraintsAndEquivClasses(fds, equivClasses);
-            }
-        } else if (funId.equals(AlgebricksBuiltinFunctions.EQ)) {
-            ILogicalExpression opLeft = arguments.get(0).getValue();
-            ILogicalExpression opRight = arguments.get(1).getValue();
-            if (opLeft.getExpressionTag() == LogicalExpressionTag.CONSTANT
-                    && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                ConstantExpression op1 = (ConstantExpression) opLeft;
-                VariableReferenceExpression op2 = (VariableReferenceExpression) opRight;
-                getFDsAndEquivClassesForEqWithConstant(op1, op2, fds, equivClasses);
-            } else if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
-                    && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression op1 = (VariableReferenceExpression) opLeft;
-                VariableReferenceExpression op2 = (VariableReferenceExpression) opRight;
-                getFDsAndEquivClassesForColumnEq(op1, op2, fds, equivClasses);
-            }
-        }
-    }
-
-    @Override
-    public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars) {
-        FunctionIdentifier funId = getFunctionIdentifier();
-        if (funId.equals(AlgebricksBuiltinFunctions.AND)) {
-            for (Mutable<ILogicalExpression> a : arguments) {
-                a.getValue().getConstraintsForOuterJoin(fds, outerVars);
-            }
-        } else if (funId.equals(AlgebricksBuiltinFunctions.EQ)) {
-            ILogicalExpression opLeft = arguments.get(0).getValue();
-            ILogicalExpression opRight = arguments.get(1).getValue();
-            if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
-                    && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
-                LogicalVariable var2 = ((VariableReferenceExpression) opRight).getVariableReference();
-                if (outerVars.contains(var1)) {
-                    addFD(fds, var1, var2);
-                }
-                if (outerVars.contains(var2)) {
-                    addFD(fds, var2, var1);
-                }
-            }
-        }
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof AbstractFunctionCallExpression)) {
-            return false;
-        } else {
-            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) obj;
-            boolean equal = getFunctionIdentifier().equals(fce.getFunctionIdentifier());
-            if (!equal)
-                return false;
-            for (int i = 0; i < arguments.size(); i++) {
-                ILogicalExpression argument = arguments.get(i).getValue();
-                ILogicalExpression fceArgument = fce.getArguments().get(i).getValue();
-                if (!argument.equals(fceArgument))
-                    return false;
-            }
-            if (opaqueParameters != null) {
-                if (opaqueParameters.length != fce.opaqueParameters.length)
-                    return false;
-                for (int i = 0; i < opaqueParameters.length; i++) {
-                    Object opaqueParameter = opaqueParameters[i];
-                    Object fceOpaqueParameter = fce.opaqueParameters[i];
-                    if (!opaqueParameter.equals(fceOpaqueParameter))
-                        return false;
-                }
-            }
-            return true;
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        int h = finfo.hashCode();
-        for (Mutable<ILogicalExpression> e : arguments) {
-            h = h * 41 + e.getValue().hashCode();
-        }
-        if (opaqueParameters != null) {
-            for (int i = 0; i < opaqueParameters.length; i++) {
-                h = h * 31 + opaqueParameters[i].hashCode();
-            }
-        }
-        return h;
-    }
-
-    @Override
-    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
-        if (!getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND) || arguments.size() <= 1) {
-            return false;
-        } else {
-            conjs.addAll(arguments);
-            return true;
-        }
-    }
-
-    public Map<Object, IExpressionAnnotation> getAnnotations() {
-        return annotationMap;
-    }
-
-    protected Map<Object, IExpressionAnnotation> cloneAnnotations() {
-        Map<Object, IExpressionAnnotation> m = new HashMap<Object, IExpressionAnnotation>();
-        for (Object k : annotationMap.keySet()) {
-            IExpressionAnnotation annot2 = annotationMap.get(k).copy();
-            m.put(k, annot2);
-        }
-        return m;
-    }
-
-    private final static void addFD(Collection<FunctionalDependency> fds, LogicalVariable var1, LogicalVariable var2) {
-        LinkedList<LogicalVariable> set1 = new LinkedList<LogicalVariable>();
-        set1.add(var1);
-        LinkedList<LogicalVariable> set2 = new LinkedList<LogicalVariable>();
-        set2.add(var2);
-        FunctionalDependency fd1 = new FunctionalDependency(set1, set2);
-        fds.add(fd1);
-    }
-
-    private final static void getFDsAndEquivClassesForEqWithConstant(ConstantExpression c,
-            VariableReferenceExpression v, Collection<FunctionalDependency> fds,
-            Map<LogicalVariable, EquivalenceClass> equivClasses) {
-        LogicalVariable var = v.getVariableReference();
-        LinkedList<LogicalVariable> head = new LinkedList<LogicalVariable>();
-        // empty set in the head
-        LinkedList<LogicalVariable> tail = new LinkedList<LogicalVariable>();
-        tail.add(var);
-        FunctionalDependency fd = new FunctionalDependency(head, tail);
-        fds.add(fd);
-
-        EquivalenceClass ec = equivClasses.get(var);
-        if (ec == null) {
-            LinkedList<LogicalVariable> members = new LinkedList<LogicalVariable>();
-            members.add(var);
-            EquivalenceClass eclass = new EquivalenceClass(members, c);
-            equivClasses.put(var, eclass);
-        } else {
-            if (ec.representativeIsConst()) {
-                ILogicalExpression c1 = ec.getConstRepresentative();
-                if (!c1.equals(c)) {
-                    // here I could also rewrite to FALSE
-                    return;
-                }
-            }
-            ec.setConstRepresentative(c);
-        }
-    }
-
-    /*
-     * Obs.: mgmt. of equiv. classes should use a more efficient data
-     * structure,if we are to implem. cost-bazed optim.
-     */
-    private final static void getFDsAndEquivClassesForColumnEq(VariableReferenceExpression v1,
-            VariableReferenceExpression v2, Collection<FunctionalDependency> fds,
-            Map<LogicalVariable, EquivalenceClass> equivClasses) {
-        LogicalVariable var1 = v1.getVariableReference();
-        LogicalVariable var2 = v2.getVariableReference();
-        LinkedList<LogicalVariable> set1 = new LinkedList<LogicalVariable>();
-        set1.add(var1);
-        LinkedList<LogicalVariable> set2 = new LinkedList<LogicalVariable>();
-        set2.add(var2);
-        FunctionalDependency fd1 = new FunctionalDependency(set1, set2);
-        FunctionalDependency fd2 = new FunctionalDependency(set2, set1);
-        fds.add(fd1);
-        fds.add(fd2);
-
-        EquivalenceClass ec1 = equivClasses.get(var1);
-        EquivalenceClass ec2 = equivClasses.get(var2);
-        if (ec1 == null && ec2 == null) {
-            LinkedList<LogicalVariable> members = new LinkedList<LogicalVariable>();
-            members.add(var1);
-            members.add(var2);
-            EquivalenceClass ec = new EquivalenceClass(members, var1);
-            equivClasses.put(var1, ec);
-            equivClasses.put(var2, ec);
-        } else if (ec1 == null && ec2 != null) {
-            ec2.addMember(var1);
-            equivClasses.put(var1, ec2);
-        } else if (ec2 == null && ec1 != null) {
-            ec1.addMember(var2);
-            equivClasses.put(var2, ec1);
-        } else {
-            ec1.merge(ec2);
-            for (LogicalVariable w : equivClasses.keySet()) {
-                if (ec2.getMembers().contains(w)) {
-                    equivClasses.put(w, ec1);
-                }
-            }
-        }
-    }
-
-    @Override
-    public boolean isFunctional() {
-        if (!finfo.isFunctional()) {
-            return false;
-        }
-
-        for (Mutable<ILogicalExpression> e : arguments) {
-            if (!e.getValue().isFunctional()) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
deleted file mode 100644
index 2718076..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.Collection;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-
-public abstract class AbstractLogicalExpression implements ILogicalExpression {
-
-    @Override
-    public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
-            Map<LogicalVariable, EquivalenceClass> equivClasses) {
-        // do nothing
-    }
-
-    @Override
-    public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds,
-            Collection<LogicalVariable> innerSideVars) {
-        // do nothing
-    }
-
-    @Override
-    public boolean isFunctional() {
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
deleted file mode 100644
index 154618b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-/**
- * 
- * An aggregate function may be executed in a "two step" mode. First the
- * "step-one" aggregates are run and then the results are passed to the
- * "step-two" aggregators. The convention is the following:
- * 
- * 1. The step-one aggregate must be able to accept the same arguments as the
- * original aggregate function call.
- * 
- * 2. The step-two aggregate must be a unary function that accepts as input the
- * output of the step-one aggregate.
- * 
- */
-
-public class AggregateFunctionCallExpression extends AbstractFunctionCallExpression {
-
-    private boolean twoStep;
-    private IFunctionInfo stepOneAggregate;
-    private IFunctionInfo stepTwoAggregate;
-
-    public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep) {
-        super(FunctionKind.AGGREGATE, finfo);
-        this.twoStep = isTwoStep;
-    }
-
-    public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
-            List<Mutable<ILogicalExpression>> arguments) {
-        super(FunctionKind.AGGREGATE, finfo, arguments);
-        this.twoStep = isTwoStep;
-    }
-
-    public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
-            Mutable<ILogicalExpression>... expressions) {
-        super(FunctionKind.AGGREGATE, finfo, expressions);
-        this.twoStep = isTwoStep;
-    }
-
-    public boolean isTwoStep() {
-        return twoStep;
-    }
-
-    public void setTwoStep(boolean twoStep) {
-        this.twoStep = twoStep;
-    }
-
-    @Override
-    public AggregateFunctionCallExpression cloneExpression() {
-        cloneAnnotations();
-        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
-        AggregateFunctionCallExpression fun = new AggregateFunctionCallExpression(finfo, twoStep, clonedArgs);
-        fun.setStepTwoAggregate(stepTwoAggregate);
-        fun.setStepOneAggregate(stepOneAggregate);
-        return fun;
-    }
-
-    public void setStepOneAggregate(IFunctionInfo stepOneAggregate) {
-        this.stepOneAggregate = stepOneAggregate;
-    }
-
-    public IFunctionInfo getStepOneAggregate() {
-        return stepOneAggregate;
-    }
-
-    public void setStepTwoAggregate(IFunctionInfo stepTwoAggregate) {
-        this.stepTwoAggregate = stepTwoAggregate;
-    }
-
-    public IFunctionInfo getStepTwoAggregate() {
-        return stepTwoAggregate;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitAggregateFunctionCallExpression(this, arg);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
deleted file mode 100644
index a29bd46..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-public class BroadcastExpressionAnnotation implements IExpressionAnnotation {
-
-    public static final String BROADCAST_ANNOTATION_KEY = "broadcast";
-
-    public enum BroadcastSide {
-        LEFT,
-        RIGHT
-    };
-
-    private BroadcastSide side;
-
-    @Override
-    public Object getObject() {
-        return side;
-    }
-
-    @Override
-    public void setObject(Object side) {
-        this.side = (BroadcastSide) side;
-    }
-
-    @Override
-    public IExpressionAnnotation copy() {
-        BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
-        bcast.side = side;
-        return bcast;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
deleted file mode 100644
index 1a3fe47..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-public final class ConstantExpression extends AbstractLogicalExpression {
-    private IAlgebricksConstantValue value;
-
-    public final static ConstantExpression TRUE = new ConstantExpression(new IAlgebricksConstantValue() {
-
-        @Override
-        public boolean isTrue() {
-            return true;
-        }
-
-        @Override
-        public boolean isNull() {
-            return false;
-        }
-
-        @Override
-        public boolean isFalse() {
-            return false;
-        }
-
-        @Override
-        public String toString() {
-            return "TRUE";
-        }
-    });
-    public final static ConstantExpression FALSE = new ConstantExpression(new IAlgebricksConstantValue() {
-
-        @Override
-        public boolean isTrue() {
-            return false;
-        }
-
-        @Override
-        public boolean isNull() {
-            return false;
-        }
-
-        @Override
-        public boolean isFalse() {
-            return true;
-        }
-
-        @Override
-        public String toString() {
-            return "FALSE";
-        }
-    });
-    public final static ConstantExpression NULL = new ConstantExpression(new IAlgebricksConstantValue() {
-
-        @Override
-        public boolean isTrue() {
-            return false;
-        }
-
-        @Override
-        public boolean isNull() {
-            return true;
-        }
-
-        @Override
-        public boolean isFalse() {
-            return false;
-        }
-
-        @Override
-        public String toString() {
-            return "NULL";
-        }
-    });
-
-    private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
-
-    public ConstantExpression(IAlgebricksConstantValue value) {
-        this.value = value;
-    }
-
-    public IAlgebricksConstantValue getValue() {
-        return value;
-    }
-
-    public void setValue(IAlgebricksConstantValue value) {
-        this.value = value;
-    }
-
-    @Override
-    public LogicalExpressionTag getExpressionTag() {
-        return LogicalExpressionTag.CONSTANT;
-    }
-
-    @Override
-    public String toString() {
-        return value.toString();
-    }
-
-    @Override
-    public void getUsedVariables(Collection<LogicalVariable> vars) {
-        // do nothing
-    }
-
-    @Override
-    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
-        // do nothing
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof ConstantExpression)) {
-            return false;
-        } else {
-            return value.equals(((ConstantExpression) obj).getValue());
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        return value.hashCode();
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitConstantExpression(this, arg);
-    }
-
-    @Override
-    public AbstractLogicalExpression cloneExpression() {
-        Map<Object, IExpressionAnnotation> m = new HashMap<Object, IExpressionAnnotation>();
-        for (Object k : annotationMap.keySet()) {
-            IExpressionAnnotation annot2 = annotationMap.get(k).copy();
-            m.put(k, annot2);
-        }
-        ConstantExpression c = new ConstantExpression(value);
-        c.annotationMap = m;
-        return c;
-    }
-
-    public Map<Object, IExpressionAnnotation> getAnnotations() {
-        return annotationMap;
-    }
-
-    @Override
-    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
-        return false;
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
deleted file mode 100644
index 975014d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-public class ExpressionAnnotationNoCopyImpl implements IExpressionAnnotation {
-
-    private Object object;
-
-    @Override
-    public IExpressionAnnotation copy() {
-        return this;
-    }
-
-    @Override
-    public Object getObject() {
-        return object;
-    }
-
-    @Override
-    public void setObject(Object object) {
-        this.object = object;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
deleted file mode 100644
index a12b8f2..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-public interface IAlgebricksConstantValue {
-    public boolean isNull();
-
-    public boolean isTrue();
-
-    public boolean isFalse();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
deleted file mode 100644
index fa2a6f4..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-public interface IExpressionAnnotation {
-    public Object getObject();
-
-    public void setObject(Object object);
-
-    public IExpressionAnnotation copy();
-}

[39/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
deleted file mode 100644
index 3b2387a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.RunningAggregateRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class RunningAggregatePOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.RUNNING_AGGREGATE;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        RunningAggregateOperator ragg = (RunningAggregateOperator) op;
-        List<LogicalVariable> variables = ragg.getVariables();
-        List<Mutable<ILogicalExpression>> expressions = ragg.getExpressions();
-        int[] outColumns = new int[variables.size()];
-        for (int i = 0; i < outColumns.length; i++) {
-            outColumns[i] = opSchema.findVariable(variables.get(i));
-        }
-        IRunningAggregateEvaluatorFactory[] runningAggFuns = new IRunningAggregateEvaluatorFactory[expressions.size()];
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        for (int i = 0; i < runningAggFuns.length; i++) {
-            StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getValue();
-            runningAggFuns[i] = expressionRuntimeProvider.createRunningAggregateFunctionFactory(expr,
-                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
-        }
-
-        // TODO push projections into the operator
-        int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
-
-        RunningAggregateRuntimeFactory runtime = new RunningAggregateRuntimeFactory(outColumns, runningAggFuns,
-                projectionList);
-
-        // contribute one Asterix framewriter
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        builder.contributeMicroOperator(ragg, runtime, recDesc);
-        // and contribute one edge from its child
-        ILogicalOperator src = ragg.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, ragg, 0);
-
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
deleted file mode 100644
index ba299ac..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.misc.SinkOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
-
-public class SinkPOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.SINK;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2;
-        List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
-        IPhysicalPropertiesVector childsProperties = null;
-
-        for (int i = 0; i < op.getInputs().size(); i++) {
-            op2 = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
-            childsProperties = op2.getPhysicalOperator().getDeliveredProperties();
-            if (childsProperties.getLocalProperties() != null) {
-                propsLocal.addAll(childsProperties.getLocalProperties());
-            }
-        }
-
-        deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-      return emptyUnaryRequirements(op.getInputs().size());
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-
-        SinkOperatorDescriptor opDesc = new SinkOperatorDescriptor(spec, op.getInputs().size());
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-
-        for (int i = 0; i < op.getInputs().size(); i++) {
-            builder.contributeGraphEdge(op.getInputs().get(i).getValue(), 0, op, i);
-        }
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
deleted file mode 100644
index 9105c1f..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class SinkWritePOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.SINK_WRITE;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        WriteOperator write = (WriteOperator) op;
-        IDataSink sink = write.getDataSink();
-        IPartitioningProperty pp = sink.getPartitioningProperty();
-        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp, null) };
-        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        WriteOperator write = (WriteOperator) op;
-        int[] columns = new int[write.getExpressions().size()];
-        int i = 0;
-        for (Mutable<ILogicalExpression> exprRef : write.getExpressions()) {
-            ILogicalExpression expr = exprRef.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new NotImplementedException("Only writing variable expressions is supported.");
-            }
-            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
-            LogicalVariable v = varRef.getVariableReference();
-            columns[i++] = inputSchemas[0].findVariable(v);
-        }
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0],
-                context);
-
-        IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
-                context, columns);
-
-        IMetadataProvider<?, ?> mp = context.getMetadataProvider();
-
-        Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> runtime = mp.getWriteFileRuntime(write.getDataSink(),
-                columns, pf, inputDesc);
-
-        builder.contributeMicroOperator(write, runtime.first, recDesc, runtime.second);
-        ILogicalOperator src = write.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, write, 0);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java
deleted file mode 100644
index 6ad1fab..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.OperatorSchemaImpl;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.sort.SortGroupByOperatorDescriptor;
-
-public class SortGroupByPOperator extends AbstractPhysicalOperator {
-
-    private final int frameLimit;
-    private final OrderColumn[] orderColumns;
-    private final List<LogicalVariable> columnSet = new ArrayList<LogicalVariable>();
-
-    public SortGroupByPOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList, int frameLimit,
-            OrderColumn[] orderColumns) {
-        this.frameLimit = frameLimit;
-        this.orderColumns = orderColumns;
-        computeColumnSet(gbyList);
-    }
-
-    private void computeColumnSet(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList) {
-        columnSet.clear();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                columnSet.add(v.getVariableReference());
-            }
-        }
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.SORT_GROUP_BY;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + columnSet;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    private List<LogicalVariable> getGbyColumns() {
-        return columnSet;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
-
-        GroupByOperator gOp = (GroupByOperator) op;
-        Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
-        List<OrderColumn> ocs = new ArrayList<OrderColumn>();
-
-        if (!columnSet.isEmpty()) {
-            propsLocal.add(new LocalGroupingProperty(columnSet));
-        }
-        for (OrderColumn oc : orderColumns) {
-            ocs.add(oc);
-        }
-        propsLocal.add(new LocalOrderProperty(ocs));
-        for (ILogicalPlan p : gOp.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                ILogicalOperator rOp = r.getValue();
-                propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
-            }
-        }
-
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
-        deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        List<LogicalVariable> gbyCols = getGbyColumns();
-        int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]);
-        GroupByOperator gby = (GroupByOperator) op;
-        int numFds = gby.getDecorList().size();
-        int fdColumns[] = new int[numFds];
-        int j = 0;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new AlgebricksException("Sort group-by expects variable references.");
-            }
-            VariableReferenceExpression v = (VariableReferenceExpression) expr;
-            LogicalVariable decor = v.getVariableReference();
-            fdColumns[j++] = inputSchemas[0].findVariable(decor);
-        }
-
-        if (gby.getNestedPlans().size() != 1) {
-            throw new AlgebricksException(
-                    "Sort group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        ILogicalPlan p0 = gby.getNestedPlans().get(0);
-        if (p0.getRoots().size() != 1) {
-            throw new AlgebricksException(
-                    "Sort group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
-        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
-
-        IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
-        List<Object> intermediateTypes = new ArrayList<Object>();
-        int n = aggOp.getExpressions().size();
-        IAggregateEvaluatorFactory[] aff = new IAggregateEvaluatorFactory[n];
-        int i = 0;
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
-        IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
-        for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
-            aff[i++] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun, aggOpInputEnv, inputSchemas,
-                    context);
-            intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
-                    context.getMetadataProvider()));
-        }
-
-        int[] keyAndDecFields = new int[keys.length + fdColumns.length];
-        for (i = 0; i < keys.length; ++i) {
-            keyAndDecFields[i] = keys[i];
-        }
-        for (i = 0; i < fdColumns.length; i++) {
-            keyAndDecFields[keys.length + i] = fdColumns[i];
-        }
-
-        List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList()) {
-            keyAndDecVariables.add(p.first);
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
-            keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
-        }
-
-        for (LogicalVariable var : keyAndDecVariables) {
-            aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
-        }
-
-        compileSubplans(inputSchemas[0], gby, opSchema, context);
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-
-        IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[gbyCols.size()];
-        IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
-        i = 0;
-        for (LogicalVariable v : gbyCols) {
-            Object type = aggOpInputEnv.getVarType(v);
-            if (orderColumns[i].getOrder() == OrderKind.ASC) {
-                compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true);
-            } else {
-                compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, false);
-            }
-            i++;
-        }
-        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema,
-                context);
-
-        IAggregateEvaluatorFactory[] merges = new IAggregateEvaluatorFactory[n];
-        List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
-        IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
-        localInputSchemas[0] = new OperatorSchemaImpl();
-        for (i = 0; i < n; i++) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
-                    .get(i).getValue();
-            aggFun.getUsedVariables(usedVars);
-        }
-        i = 0;
-        for (Object type : intermediateTypes) {
-            aggOpInputEnv.setVarType(usedVars.get(i++), type);
-        }
-        for (LogicalVariable keyVar : keyAndDecVariables) {
-            localInputSchemas[0].addVariable(keyVar);
-        }
-        for (LogicalVariable usedVar : usedVars) {
-            localInputSchemas[0].addVariable(usedVar);
-        }
-        for (i = 0; i < n; i++) {
-            AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
-                    .get(i).getValue();
-            merges[i] = expressionRuntimeProvider.createAggregateFunctionFactory(mergeFun, aggOpInputEnv,
-                    localInputSchemas, context);
-        }
-        RecordDescriptor partialAggRecordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
-                localInputSchemas[0], context);
-
-        IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aff,
-                keyAndDecFields);
-        IAggregatorDescriptorFactory mergeFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(merges,
-                keyAndDecFields);
-
-        INormalizedKeyComputerFactory normalizedKeyFactory = null;
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        if (nkcfProvider == null) {
-            normalizedKeyFactory = null;
-        }
-        Object type = aggOpInputEnv.getVarType(gbyCols.get(0));
-        normalizedKeyFactory = orderColumns[0].getOrder() == OrderKind.ASC ? nkcfProvider
-                .getNormalizedKeyComputerFactory(type, true) : nkcfProvider
-                .getNormalizedKeyComputerFactory(type, false);
-        SortGroupByOperatorDescriptor gbyOpDesc = new SortGroupByOperatorDescriptor(spec, frameLimit, keys,
-                keyAndDecFields, normalizedKeyFactory, compFactories, aggregatorFactory, mergeFactory,
-                partialAggRecordDescriptor, recordDescriptor, false);
-
-        contributeOpDesc(builder, gby, gbyOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[] { 0 };
-        int[] outputDependencyLabels = new int[] { 1 };
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
deleted file mode 100644
index 7f29375..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
-
-public class SortMergeExchangePOperator extends AbstractExchangePOperator {
-
-    private final OrderColumn[] sortColumns;
-
-    public SortMergeExchangePOperator(OrderColumn[] sortColumns) {
-        this.sortColumns = sortColumns;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.SORT_MERGE_EXCHANGE;
-    }
-
-    public OrderColumn[] getSortColumns() {
-        return sortColumns;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append(getOperatorTag());
-        sb.append(" [");
-        sb.append(sortColumns[0]);
-        for (int i = 1; i < sortColumns.length; i++) {
-            sb.append(", " + sortColumns[i]);
-        }
-        sb.append(" ]");
-        return sb.toString();
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator inp1 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        IPhysicalPropertiesVector pv1 = inp1.getDeliveredPhysicalProperties();
-        if (pv1 == null) {
-            inp1.computeDeliveredPhysicalProperties(context);
-            pv1 = inp1.getDeliveredPhysicalProperties();
-        }
-
-        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-        List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(sortColumns.length);
-        for (ILocalStructuralProperty prop : pv1.getLocalProperties()) {
-            if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
-                LocalOrderProperty lop = (LocalOrderProperty) prop;
-                for (OrderColumn oc : lop.getOrderColumns()) {
-                    if (oc.equals(sortColumns[orderColumns.size()])) {
-                        orderColumns.add(oc);
-                        if (orderColumns.size() == sortColumns.length) {
-                            break;
-                        }
-                    } else {
-                        break;
-                    }
-                }
-            } else {
-                continue;
-            }
-        }
-        if (orderColumns.size() > 0) {
-            localProps.add(new LocalOrderProperty(orderColumns));
-        }
-        this.deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, localProps);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(sortColumns.length);
-        localProps.add(new LocalOrderProperty(Arrays.asList(sortColumns)));
-        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
-                localProps) };
-        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        int n = sortColumns.length;
-        int[] sortFields = new int[n];
-        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
-        IBinaryHashFunctionFactory[] hashFuns = new IBinaryHashFunctionFactory[n];
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        INormalizedKeyComputerFactory nkcf = null;
-
-        for (int i = 0; i < n; i++) {
-            sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn());
-            Object type = env.getVarType(sortColumns[i].getColumn());
-            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-            comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
-            IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider();
-            hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type);
-            if (i == 0 && nkcfProvider != null && type != null) {
-                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
-            }
-        }
-        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(sortFields, hashFuns);
-        IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
deleted file mode 100644
index 2c4d89d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-
-/**
- * This will always be attached to an {@link OrderOperator} logical operator.
- */
-
-public class StableSortPOperator extends AbstractStableSortPOperator {
-
-    private int maxNumberOfFrames;
-
-    public StableSortPOperator(int maxNumberOfFrames) {
-        super();
-        this.maxNumberOfFrames = maxNumberOfFrames;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.STABLE_SORT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        int n = sortColumns.length;
-        int[] sortFields = new int[n];
-        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
-
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        INormalizedKeyComputerFactory nkcf = null;
-
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        int i = 0;
-        for (OrderColumn oc : sortColumns) {
-            LogicalVariable var = oc.getColumn();
-            sortFields[i] = opSchema.findVariable(var);
-            Object type = env.getVarType(var);
-            OrderKind order = oc.getOrder();
-            if (i == 0 && nkcfProvider != null && type != null) {
-                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
-            }
-            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
-            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
-            i++;
-        }
-
-        ExternalSortOperatorDescriptor sortOpDesc = new ExternalSortOperatorDescriptor(spec, maxNumberOfFrames,
-                sortFields, nkcf, comps, recDescriptor);
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
deleted file mode 100644
index 63ddffe..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamLimitRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class StreamLimitPOperator extends AbstractPhysicalOperator {
-
-    public StreamLimitPOperator() {
-
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.STREAM_LIMIT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator limitOp = (AbstractLogicalOperator) op;
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        if (limitOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
-            //partitioning property: unpartitioned;  local property: whatever from the child
-            deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, op2
-                    .getDeliveredPhysicalProperties().getLocalProperties());
-        } else {
-            deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-        }
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        AbstractLogicalOperator limitOp = (AbstractLogicalOperator) op;
-        if (limitOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
-            StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
-            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
-            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-        } else {
-            return emptyUnaryRequirements();
-        }
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        LimitOperator limit = (LimitOperator) op;
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        IScalarEvaluatorFactory maxObjectsFact = expressionRuntimeProvider.createEvaluatorFactory(limit.getMaxObjects()
-                .getValue(), env, inputSchemas, context);
-        ILogicalExpression offsetExpr = limit.getOffset().getValue();
-        IScalarEvaluatorFactory offsetFact = (offsetExpr == null) ? null : expressionRuntimeProvider
-                .createEvaluatorFactory(offsetExpr, env, inputSchemas, context);
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
-                context);
-        StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null,
-                context.getBinaryIntegerInspectorFactory());
-        builder.contributeMicroOperator(limit, runtime, recDesc);
-        // and contribute one edge from its child
-        ILogicalOperator src = limit.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, limit, 0);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
deleted file mode 100644
index 2ad5fda..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamProjectRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class StreamProjectPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
-
-    private boolean flushFramesRapidly;
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.STREAM_PROJECT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        ProjectOperator project = (ProjectOperator) op;
-        int[] projectionList = new int[project.getVariables().size()];
-        int i = 0;
-        for (LogicalVariable v : project.getVariables()) {
-            int pos = inputSchemas[0].findVariable(v);
-            if (pos < 0) {
-                throw new AlgebricksException("Could not find variable " + v + ".");
-            }
-            projectionList[i++] = pos;
-        }
-        StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList, flushFramesRapidly);
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
-                context);
-        builder.contributeMicroOperator(project, runtime, recDesc);
-        ILogicalOperator src = project.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, project, 0);
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ProjectOperator p = (ProjectOperator) op;
-        computeDeliveredPropertiesForUsedVariables(p, p.getVariables());
-    }
-
-    public void setRapidFrameFlush(boolean flushFramesRapidly) {
-        this.flushFramesRapidly = flushFramesRapidly;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
deleted file mode 100644
index 3999fa6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamSelectRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class StreamSelectPOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.STREAM_SELECT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        SelectOperator select = (SelectOperator) op;
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(select.getCondition()
-                .getValue(), context.getTypeEnvironment(op), inputSchemas, context);
-        StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null,
-                context.getBinaryBooleanInspectorFactory(), select.getRetainNull(), inputSchemas[0].findVariable(select
-                        .getNullPlaceholderVariable()), context.getNullWriterFactory());
-        // contribute one Asterix framewriter
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        builder.contributeMicroOperator(select, runtime, recDesc);
-        // and contribute one edge from its child
-        ILogicalOperator src = select.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, select, 0);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
deleted file mode 100644
index e6a0be7..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
-import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription.ScriptKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.StringStreamingScriptDescription;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StringStreamingRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class StringStreamingScriptPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.STRING_STREAM_SCRIPT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        ScriptOperator scriptOp = (ScriptOperator) op;
-        IScriptDescription scriptDesc = scriptOp.getScriptDescription();
-        if (scriptDesc.getKind() != ScriptKind.STRING_STREAMING) {
-            throw new IllegalStateException();
-        }
-        StringStreamingScriptDescription sssd = (StringStreamingScriptDescription) scriptDesc;
-        StringStreamingRuntimeFactory runtime = new StringStreamingRuntimeFactory(sssd.getCommand(),
-                sssd.getPrinterFactories(), sssd.getFieldDelimiter(), sssd.getParserFactory());
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
-                context);
-        builder.contributeMicroOperator(scriptOp, runtime, recDesc);
-        // and contribute one edge from its child
-        ILogicalOperator src = scriptOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, scriptOp, 0);
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ScriptOperator s = (ScriptOperator) op;
-        computeDeliveredPropertiesForUsedVariables(s, s.getInputVariables());
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
deleted file mode 100644
index efc609a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.SubplanRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class SubplanPOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.SUBPLAN;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        IPhysicalPropertiesVector childsProperties = op2.getPhysicalOperator().getDeliveredProperties();
-        List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
-        if (childsProperties.getLocalProperties() != null) {
-            propsLocal.addAll(childsProperties.getLocalProperties());
-        }
-        // ... get local properties for newly created variables...
-        SubplanOperator subplan = (SubplanOperator) op;
-        for (ILogicalPlan plan : subplan.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
-                AbstractLogicalOperator rOp = (AbstractLogicalOperator) r.getValue();
-                propsLocal.addAll(rOp.getPhysicalOperator().getDeliveredProperties().getLocalProperties());
-            }
-        }
-
-        deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        SubplanOperator subplan = (SubplanOperator) op;
-        if (subplan.getNestedPlans().size() != 1) {
-            throw new NotImplementedException("Subplan currently works only for one nested plan with one root.");
-        }
-        AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], subplan, opSchema, context);
-        assert (subplans.length == 1);
-        AlgebricksPipeline np = subplans[0];
-        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()),
-                inputSchemas[0], context);
-        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[np.getOutputWidth()];
-        for (int i = 0; i < nullWriterFactories.length; i++) {
-            nullWriterFactories[i] = context.getNullWriterFactory();
-        }
-        SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, nullWriterFactories, inputRecordDesc, null);
-
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        builder.contributeMicroOperator(subplan, runtime, recDesc);
-
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java
deleted file mode 100644
index 8a2ecf6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class TokenizePOperator extends AbstractPhysicalOperator {
-
-    private final List<LogicalVariable> primaryKeys;
-    private final List<LogicalVariable> secondaryKeys;
-    private final IDataSourceIndex<?, ?> dataSourceIndex;
-
-    public TokenizePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            IDataSourceIndex<?, ?> dataSourceIndex) {
-        this.primaryKeys = primaryKeys;
-        this.secondaryKeys = secondaryKeys;
-        this.dataSourceIndex = dataSourceIndex;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.TOKENIZE;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        TokenizeOperator tokenizeOp = (TokenizeOperator) op;
-        if (tokenizeOp.getOperation() != Kind.INSERT || !tokenizeOp.isBulkload()) {
-            throw new AlgebricksException("Tokenize Operator only works when bulk-loading data.");
-        }
-
-        IMetadataProvider mp = context.getMetadataProvider();
-        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
-        JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getTokenizerRuntime(
-                dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, null, inputDesc,
-                context, spec, true);
-        builder.contributeHyracksOperator(tokenizeOp, runtimeAndConstraints.first);
-        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = tokenizeOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, tokenizeOp, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
deleted file mode 100644
index cda3b00..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
-
-public class UnionAllPOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.UNION_ALL;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
-        this.deliveredProperties = new StructuralPropertiesVector(pp, new ArrayList<ILocalStructuralProperty>(0));
-
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        StructuralPropertiesVector pv0 = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
-        StructuralPropertiesVector pv1 = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
-        return new PhysicalRequirements(new StructuralPropertiesVector[] { pv0, pv1 },
-                IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        UnionAllOperator unionOp = (UnionAllOperator) op;
-        int n = unionOp.getVariableMappings().size();
-        int[] leftColumns = new int[n];
-        int[] rightColumns = new int[n];
-        int i = 0;
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : unionOp.getVariableMappings()) {
-            int posLeft = inputSchemas[0].findVariable(t.first);
-            leftColumns[i] = posLeft;
-            int posRight = inputSchemas[1].findVariable(t.second);
-            rightColumns[i] = posRight;
-            ++i;
-        }
-
-        IOperatorDescriptorRegistry spec = builder.getJobSpec();
-        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-
-        // at algebricks level, union all only accepts two inputs, although at
-        // hyracks
-        // level, there is no restrictions
-        UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(spec, 2, recordDescriptor);
-        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-        ILogicalOperator src1 = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getValue();
-        builder.contributeGraphEdge(src2, 0, op, 1);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

[47/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
deleted file mode 100644
index 528ae4e..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-
-public abstract class AbstractLogicalOperator implements ILogicalOperator {
-
-    /*********************************************************************
-     * UNPARTITIONED, the input data is not partitioned
-     * PARTITIONED, the input data is partitioned, the operator is executed on
-     * each partition and may receive input from other partitions (e.g. if it is
-     * a join or an aggregate)
-     * LOCAL, the input data is partitioned, the operator is executed on each
-     * partition and only processes data from that partition
-     */
-
-    public static enum ExecutionMode {
-        UNPARTITIONED,
-        PARTITIONED,
-        LOCAL
-    }
-
-    private AbstractLogicalOperator.ExecutionMode mode = AbstractLogicalOperator.ExecutionMode.UNPARTITIONED;
-    protected IPhysicalOperator physicalOperator;
-    private final Map<String, Object> annotations = new HashMap<String, Object>();
-    private boolean bJobGenEnabled = true;
-
-    final protected List<Mutable<ILogicalOperator>> inputs;
-    // protected List<LogicalOperatorReference> outputs;
-    protected List<LogicalVariable> schema;
-
-    public AbstractLogicalOperator() {
-        inputs = new ArrayList<Mutable<ILogicalOperator>>();
-        // outputs = new ArrayList<LogicalOperatorReference>();
-    }
-
-    @Override
-    public abstract LogicalOperatorTag getOperatorTag();
-
-    public ExecutionMode getExecutionMode() {
-        return mode;
-    }
-
-    public void setExecutionMode(ExecutionMode mode) {
-        this.mode = mode;
-    }
-
-    @Override
-    public List<LogicalVariable> getSchema() {
-        return schema;
-    }
-
-    public void setPhysicalOperator(IPhysicalOperator physicalOp) {
-        this.physicalOperator = physicalOp;
-    }
-
-    public IPhysicalOperator getPhysicalOperator() {
-        return physicalOperator;
-    }
-
-    /**
-     * @return for each child, one vector of required physical properties
-     */
-
-    @Override
-    public final PhysicalRequirements getRequiredPhysicalPropertiesForChildren(
-            IPhysicalPropertiesVector requiredProperties) {
-        return physicalOperator.getRequiredPropertiesForChildren(this, requiredProperties);
-    }
-
-    /**
-     * @return the physical properties that this operator delivers, based on
-     *         what its children deliver
-     */
-
-    @Override
-    public final IPhysicalPropertiesVector getDeliveredPhysicalProperties() {
-        return physicalOperator.getDeliveredProperties();
-    }
-
-    @Override
-    public final void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException {
-        physicalOperator.computeDeliveredProperties(this, context);
-    }
-
-    @Override
-    public final List<Mutable<ILogicalOperator>> getInputs() {
-        return inputs;
-    }
-
-    // @Override
-    // public final List<LogicalOperatorReference> getOutputs() {
-    // return outputs;
-    // }
-
-    @Override
-    public final boolean hasInputs() {
-        return !inputs.isEmpty();
-    }
-
-    public boolean hasNestedPlans() {
-        return false;
-    }
-
-    @Override
-    public Map<String, Object> getAnnotations() {
-        return annotations;
-    }
-
-    @Override
-    public void removeAnnotation(String annotationName) {
-        annotations.remove(annotationName);
-    }
-
-    @Override
-    public final void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        if (bJobGenEnabled) {
-            if (physicalOperator == null) {
-                throw new AlgebricksException("Physical operator not set for operator: " + this);
-            }
-            physicalOperator.contributeRuntimeOperator(builder, context, this, propagatedSchema, inputSchemas,
-                    outerPlanSchema);
-        }
-    }
-
-    public void disableJobGen() {
-        bJobGenEnabled = false;
-    }
-
-    public boolean isJobGenEnabled() {
-        return bJobGenEnabled;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    protected PropagatingTypeEnvironment createPropagatingAllInputsTypeEnvironment(ITypingContext ctx) {
-        int n = inputs.size();
-        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
-        for (int i = 0; i < n; i++) {
-            envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
-        }
-        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
-                ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
-    }
-
-    @Override
-    public boolean requiresVariableReferenceExpressions() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
deleted file mode 100644
index 417c959..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public abstract class AbstractOperatorWithNestedPlans extends AbstractLogicalOperator {
-    protected final List<ILogicalPlan> nestedPlans;
-
-    public AbstractOperatorWithNestedPlans() {
-        nestedPlans = new ArrayList<ILogicalPlan>();
-    }
-
-    public AbstractOperatorWithNestedPlans(List<ILogicalPlan> nestedPlans) {
-        this.nestedPlans = nestedPlans;
-    }
-
-    public List<ILogicalPlan> getNestedPlans() {
-        return nestedPlans;
-    }
-
-    @Override
-    public boolean hasNestedPlans() {
-        return true;
-    }
-
-    public LinkedList<Mutable<ILogicalOperator>> allRootsInReverseOrder() {
-        LinkedList<Mutable<ILogicalOperator>> allRoots = new LinkedList<Mutable<ILogicalOperator>>();
-        for (ILogicalPlan p : nestedPlans) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                allRoots.addFirst(r);
-            }
-        }
-        return allRoots;
-    }
-
-    //
-    // @Override
-    // public void computeConstraintsAndEquivClasses() {
-    // for (ILogicalPlan p : nestedPlans) {
-    // for (LogicalOperatorReference r : p.getRoots()) {
-    // AbstractLogicalOperator op = (AbstractLogicalOperator) r.getOperator();
-    // equivalenceClasses.putAll(op.getEquivalenceClasses());
-    // functionalDependencies.addAll(op.getFDs());
-    // }
-    // }
-    // }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-        for (ILogicalPlan p : nestedPlans) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                schema.addAll(r.getValue().getSchema());
-            }
-        }
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    public abstract void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars);
-
-    public abstract void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
deleted file mode 100644
index e18fbfb..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-
-public abstract class AbstractScanOperator extends AbstractLogicalOperator {
-    protected List<LogicalVariable> variables;
-
-    public AbstractScanOperator(List<LogicalVariable> variables) {
-        this.variables = variables;
-    }
-
-    public List<LogicalVariable> getVariables() {
-        return variables;
-    }
-
-    public void setVariables(List<LogicalVariable> variables) {
-        this.variables = variables;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-        schema.addAll(variables);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                if (sources.length > 0) {
-                    target.addAllVariables(sources[0]);
-                }
-                for (LogicalVariable v : variables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
deleted file mode 100644
index fdc827f..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-
-public abstract class AbstractUnnestOperator extends AbstractScanOperator {
-
-    protected final Mutable<ILogicalExpression> expression;
-
-    public AbstractUnnestOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression) {
-        super(variables);
-        this.expression = expression;
-    }
-
-    public Mutable<ILogicalExpression> getExpressionRef() {
-        return expression;
-    }
-
-    @Override
-    public boolean isMap() {
-        return true;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return visitor.transform(expression);
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
deleted file mode 100644
index 371da4c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class AggregateOperator extends AbstractAssignOperator {
-
-    // private ArrayList<AggregateFunctionCallExpression> expressions;
-    // TODO type safe list of expressions
-    private List<Mutable<ILogicalExpression>> mergeExpressions;
-    private boolean global;
-
-    public AggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
-        super(variables, expressions);
-        global = true;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.AGGREGATE;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitAggregateOperator(this, arg);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                for (LogicalVariable v : variables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(variables);
-    }
-
-    public void setMergeExpressions(List<Mutable<ILogicalExpression>> merges) {
-        mergeExpressions = merges;
-    }
-
-    public List<Mutable<ILogicalExpression>> getMergeExpressions() {
-        return mergeExpressions;
-    }
-
-    public void setGlobal(boolean global) {
-        this.global = global;
-    }
-
-    public boolean isGlobal() {
-        return global;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-                ctx.getMetadataProvider());
-        IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            Object t = ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(),
-                    ctx.getMetadataProvider(), env2);
-            env.setVarType(variables.get(i), t);
-        }
-        return env;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
deleted file mode 100644
index 9a8c428..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.List;
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-/**
- * It corresponds to the Map operator in other algebras.
- * 
- * @author Nicola
- */
-
-public class AssignOperator extends AbstractAssignOperator {
-	
-	private LocalOrderProperty explicitOrderingProperty;
-
-    public AssignOperator(List<LogicalVariable> vars, List<Mutable<ILogicalExpression>> exprs) {
-        super(vars, exprs);
-    }
-
-    public AssignOperator(LogicalVariable var, Mutable<ILogicalExpression> expr) {
-        super();
-        this.variables.add(var);
-        this.expressions.add(expr);
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.ASSIGN;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitAssignOperator(this, arg);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                target.addAllVariables(sources[0]);
-                for (LogicalVariable v : variables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-
-    }
-
-    @Override
-    public boolean isMap() {
-        return true;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        PropagatingTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            env.setVarType(
-                    variables.get(i),
-                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
-                            env));
-            if (expressions.get(i).getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                LogicalVariable var = ((VariableReferenceExpression) expressions.get(i).getValue())
-                        .getVariableReference();
-                for (List<LogicalVariable> list : env.getCorrelatedNullableVariableLists()) {
-                    if (list.contains(var)) {
-                        list.add(variables.get(i));
-                    }
-                }
-            }
-        }
-        return env;
-    }
-
-    @Override
-    public boolean requiresVariableReferenceExpressions() {
-        return false;
-    }
-
-    public LocalOrderProperty getExplicitOrderingProperty() {
-		return explicitOrderingProperty;
-	}
-
-	public void setExplicitOrderingProperty(
-			LocalOrderProperty explicitOrderingProperty) {
-		this.explicitOrderingProperty = explicitOrderingProperty;
-	}
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
deleted file mode 100644
index ff21478..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class DataSourceScanOperator extends AbstractDataSourceOperator {
-    private List<LogicalVariable> projectVars;
-
-    private boolean projectPushed = false;
-
-    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
-    private List<LogicalVariable> minFilterVars;
-    private List<LogicalVariable> maxFilterVars;
-
-    public DataSourceScanOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
-        super(variables, dataSource);
-        projectVars = new ArrayList<LogicalVariable>();
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.DATASOURCESCAN;
-    }
-
-    @Override
-    public <R, S> R accept(ILogicalOperatorVisitor<R, S> visitor, S arg) throws AlgebricksException {
-        return visitor.visitDataScanOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    public void addProjectVariables(Collection<LogicalVariable> vars) {
-        projectVars.addAll(vars);
-        projectPushed = true;
-    }
-
-    public List<LogicalVariable> getProjectVariables() {
-        return projectVars;
-    }
-
-    public boolean isProjectPushed() {
-        return projectPushed;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                if (sources.length > 0) {
-                    target.addAllVariables(sources[0]);
-                }
-                List<LogicalVariable> outputVariables = projectPushed ? projectVars : variables;
-                for (LogicalVariable v : outputVariables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
-        Object[] types = dataSource.getSchemaTypes();
-        int i = 0;
-        for (LogicalVariable v : variables) {
-            env.setVarType(v, types[i]);
-            ++i;
-        }
-        return env;
-    }
-
-    public List<LogicalVariable> getMinFilterVars() {
-        return minFilterVars;
-    }
-
-    public void setMinFilterVars(List<LogicalVariable> minFilterVars) {
-        this.minFilterVars = minFilterVars;
-    }
-
-    public List<LogicalVariable> getMaxFilterVars() {
-        return maxFilterVars;
-    }
-
-    public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) {
-        this.maxFilterVars = maxFilterVars;
-    }
-
-    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
-        this.additionalFilteringExpressions = additionalFilteringExpressions;
-    }
-
-    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
-        return additionalFilteringExpressions;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
deleted file mode 100644
index 3d8c4b6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class DistinctOperator extends AbstractLogicalOperator {
-    private final List<Mutable<ILogicalExpression>> expressions;
-
-    public DistinctOperator(List<Mutable<ILogicalExpression>> expressions) {
-        this.expressions = expressions;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.DISTINCT;
-    }
-
-    public List<Mutable<ILogicalExpression>> getExpressions() {
-        return expressions;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(this.getDistinctByVarList());
-        List<LogicalVariable> inputSchema = inputs.get(0).getValue().getSchema();
-        for (LogicalVariable var : inputSchema) {
-            if (!schema.contains(var)) {
-                schema.add(var);
-            }
-        }
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                /** make sure distinct key vars laid-out first */
-                for (LogicalVariable keyVar : getDistinctByVarList()) {
-                    target.addVariable(keyVar);
-                }
-                /** add other source vars */
-                for (IOperatorSchema srcSchema : sources) {
-                    for (LogicalVariable srcVar : srcSchema)
-                        if (target.findVariable(srcVar) < 0) {
-                            target.addVariable(srcVar);
-                        }
-                }
-            }
-        };
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean changed = false;
-        for (Mutable<ILogicalExpression> e : expressions) {
-            if (visitor.transform(e)) {
-                changed = true;
-            }
-        }
-        return changed;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitDistinctOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    public List<LogicalVariable> getDistinctByVarList() {
-        List<LogicalVariable> varList = new ArrayList<LogicalVariable>(expressions.size());
-        for (Mutable<ILogicalExpression> eRef : expressions) {
-            ILogicalExpression e = eRef.getValue();
-            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) e;
-                varList.add(v.getVariableReference());
-            }
-        }
-        return varList;
-    }
-
-    public boolean isDistinctByVar(LogicalVariable var) {
-        for (Mutable<ILogicalExpression> eRef : expressions) {
-            ILogicalExpression e = eRef.getValue();
-            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) e;
-                if (v.getVariableReference() == var) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java
deleted file mode 100644
index 3e28d73..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistributeResultOperator.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class DistributeResultOperator extends AbstractLogicalOperator {
-    private List<Mutable<ILogicalExpression>> expressions;
-    private IDataSink dataSink;
-
-    public DistributeResultOperator(List<Mutable<ILogicalExpression>> expressions, IDataSink dataSink) {
-        this.expressions = expressions;
-        this.dataSink = dataSink;
-    }
-
-    public List<Mutable<ILogicalExpression>> getExpressions() {
-        return expressions;
-    }
-
-    public IDataSink getDataSink() {
-        return dataSink;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.DISTRIBUTE_RESULT;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitDistributeResultOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean modif = false;
-        for (int i = 0; i < expressions.size(); i++) {
-            boolean b = visitor.transform(expressions.get(i));
-            if (b) {
-                modif = true;
-            }
-        }
-        return modif;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return false; // actually depends on the physical op.
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
deleted file mode 100644
index 7d16ed1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class EmptyTupleSourceOperator extends AbstractLogicalOperator {
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.EMPTYTUPLESOURCE;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.NONE;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
-        // do nothing
-        return false;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitEmptyTupleSourceOperator(this, arg);
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException {
-        return new IVariableTypeEnvironment() {
-
-            @Override
-            public void setVarType(LogicalVariable var, Object type) {
-                throw new IllegalStateException();
-            }
-
-            @Override
-            public Object getVarType(LogicalVariable var) throws AlgebricksException {
-                return null;
-            }
-
-            @Override
-            public Object getType(ILogicalExpression expr) throws AlgebricksException {
-                return ctx.getExpressionTypeComputer().getType(expr, ctx.getMetadataProvider(), this);
-            }
-
-            @Override
-            public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
-                    List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
-                return null;
-            }
-
-            @Override
-            public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2)
-                    throws AlgebricksException {
-                return false;
-            }
-        };
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
deleted file mode 100644
index a3c2650..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ExchangeOperator extends AbstractLogicalOperator {
-
-    public ExchangeOperator() {
-        super();
-        setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitExchangeOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
-        // do nothing
-        return false;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.EXCHANGE;
-    }
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        AbstractLogicalOperator cld = (AbstractLogicalOperator) inputs.get(0).getValue();
-        OperatorPropertiesUtil.computeSchemaRecIfNull(cld);
-        List<LogicalVariable> inputSchema = inputs.get(0).getValue().getSchema();
-        schema = new ArrayList<LogicalVariable>(inputSchema);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return true;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
deleted file mode 100644
index bdf6f1c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExtensionOperator.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-/**
- * @author rico
- */
-public class ExtensionOperator extends AbstractLogicalOperator {
-
-    private IOperatorExtension delegate;
-
-    public ExtensionOperator(IOperatorExtension delegate) {
-        super();
-        if (delegate == null) {
-            throw new IllegalArgumentException("delegate cannot be null!");
-        }
-        this.delegate = delegate;
-        setExecutionMode(delegate.getExecutionMode());
-    }
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
-        delegate.setSchema(schema);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
-        return delegate.acceptExpressionTransform(transform);
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitExtensionOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return this.delegate.isMap();
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return this.createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.EXTENSION_OPERATOR;
-    }
-
-    public IOperatorExtension getNewInstanceOfDelegateOperator() {
-        return delegate.newInstance();
-    }
-
-    @Override
-    public List<LogicalVariable> getSchema() {
-        return this.schema;
-    }
-
-    @Override
-    public ExecutionMode getExecutionMode() {
-        return delegate.getExecutionMode();
-    }
-
-    @Override
-    public void setExecutionMode(ExecutionMode mode) {
-        delegate.setExecutionMode(mode);
-    }
-
-    @Override
-    public IPhysicalOperator getPhysicalOperator() {
-        return delegate.getPhysicalOperator();
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return this.createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    @Override
-    public String toString() {
-        return delegate.toString();
-    }
-    
-    public IOperatorExtension getDelegate() {
-        return delegate;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
deleted file mode 100644
index 4737cfc..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExternalDataLookupOperator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ExternalDataLookupOperator extends AbstractDataSourceOperator {
-
-    private final List<Object> variableTypes;
-    protected final Mutable<ILogicalExpression> expression;
-    private final boolean propagateInput;
-
-    public ExternalDataLookupOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
-            List<Object> variableTypes, boolean propagateInput, IDataSource<?> dataSource) {
-        super(variables, dataSource);
-        this.expression = expression;
-        this.variableTypes = variableTypes;
-        this.propagateInput = propagateInput;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.EXTERNAL_LOOKUP;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = null;
-        if (propagateInput) {
-            env = createPropagatingAllInputsTypeEnvironment(ctx);
-        } else {
-            env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMetadataProvider());
-        }
-        env.setVarType(variables.get(0), variableTypes.get(0));
-        return env;
-    }
-
-    public List<Object> getVariableTypes() {
-        return variableTypes;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitExternalDataLookupOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    public boolean isPropagateInput() {
-        return propagateInput;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                if (propagateInput) {
-                    ArrayList<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
-                    VariableUtilities.getUsedVariables(ExternalDataLookupOperator.this, usedVariables);
-                    int numOfSources = sources.length;
-                    for (int i = 0; i < numOfSources; i++) {
-                        for (LogicalVariable v : sources[i]) {
-                            if (!usedVariables.contains(v)) {
-                                target.addVariable(v);
-                            }
-                        }
-                    }
-                }
-                target.addVariable(variables.get(0));
-            }
-        };
-    }
-
-    public Mutable<ILogicalExpression> getExpressionRef() {
-        return expression;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return visitor.transform(expression);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
deleted file mode 100644
index a5b7074..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class GroupByOperator extends AbstractOperatorWithNestedPlans {
-    // If the LogicalVariable in a pair is null, it means that the GroupBy is
-    // only grouping by the expression, without producing a new variable.
-    private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList;
-    private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList;
-
-    // In decorList, if the variable (first member of the pair) is null, the
-    // second member of the pair is variable reference which is propagated.
-
-    public GroupByOperator() {
-        super();
-        gByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
-        decorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
-    }
-
-    public GroupByOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList,
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList, List<ILogicalPlan> nestedPlans) {
-        super(nestedPlans);
-        this.decorList = decorList;
-        this.gByList = groupByList;
-    }
-
-    public void addGbyExpression(LogicalVariable variable, ILogicalExpression expression) {
-        this.gByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
-                new MutableObject<ILogicalExpression>(expression)));
-    }
-
-    public void addDecorExpression(LogicalVariable variable, ILogicalExpression expression) {
-        this.decorList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
-                new MutableObject<ILogicalExpression>(expression)));
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.GROUP;
-    }
-
-    public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getGroupByList() {
-        return gByList;
-    }
-
-    public String gByListToString() {
-        return veListToString(gByList);
-    }
-
-    public String decorListToString() {
-        return veListToString(decorList);
-    }
-
-    public List<LogicalVariable> getGbyVarList() {
-        List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size());
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) {
-            ILogicalExpression expr = ve.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                varList.add(v.getVariableReference());
-            }
-        }
-        return varList;
-    }
-
-    public List<LogicalVariable> getProducedGbyVarList() {
-        List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size());
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) {
-            varList.add(ve.first);
-        }
-        return varList;
-    }
-
-    public static String veListToString(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList) {
-        StringBuilder sb = new StringBuilder();
-        sb.append("[");
-        boolean fst = true;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
-            if (fst) {
-                fst = false;
-            } else {
-                sb.append("; ");
-            }
-            if (ve.first != null) {
-                sb.append(ve.first + " := " + ve.second);
-            } else {
-                sb.append(ve.second.getValue());
-            }
-        }
-        sb.append("]");
-        return sb.toString();
-    }
-
-    @Override
-    public void recomputeSchema() {
-        super.recomputeSchema();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-            schema.add(p.first);
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
-            schema.add(getDecorVariable(p));
-        }
-    }
-
-    @Override
-    public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
-        // super.getProducedVariables(vars);
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-            if (p.first != null) {
-                vars.add(p.first);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
-            if (p.first != null) {
-                vars.add(p.first);
-            }
-        }
-    }
-
-    @Override
-    public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : gByList) {
-            g.second.getValue().getUsedVariables(vars);
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : decorList) {
-            g.second.getValue().getUsedVariables(vars);
-        }
-        // super.getUsedVariables(vars);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-                    ILogicalExpression expr = p.second.getValue();
-                    if (p.first != null) {
-                        target.addVariable(p.first);
-                    } else {
-                        if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                            throw new AlgebricksException("hash group-by expects variable references.");
-                        }
-                        VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                        target.addVariable(v.getVariableReference());
-                    }
-                }
-                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
-                    ILogicalExpression expr = p.second.getValue();
-                    if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                        throw new AlgebricksException("pre-sorted group-by expects variable references.");
-                    }
-                    VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                    LogicalVariable decor = v.getVariableReference();
-                    if (p.first != null) {
-                        target.addVariable(p.first);
-                    } else {
-                        target.addVariable(decor);
-                    }
-                }
-
-            }
-        };
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean b = false;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-            if (visitor.transform(p.second)) {
-                b = true;
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
-            if (visitor.transform(p.second)) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitGroupByOperator(this, arg);
-    }
-
-    public static LogicalVariable getDecorVariable(Pair<LogicalVariable, Mutable<ILogicalExpression>> p) {
-        if (p.first != null) {
-            return p.first;
-        } else {
-            VariableReferenceExpression e = (VariableReferenceExpression) p.second.getValue();
-            return e.getVariableReference();
-        }
-    }
-
-    public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getDecorList() {
-        return decorList;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        int n = 0;
-        for (ILogicalPlan p : nestedPlans) {
-            n += p.getRoots().size();
-        }
-        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
-        int i = 0;
-        for (ILogicalPlan p : nestedPlans) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
-                i++;
-            }
-        }
-        IVariableTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
-        ILogicalOperator child = inputs.get(0).getValue();
-        IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(child);
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getGroupByList()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (p.first != null) {
-                env.setVarType(p.first, env2.getType(expr));
-                if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                    LogicalVariable v1 = ((VariableReferenceExpression) expr).getVariableReference();
-                    env.setVarType(v1, env2.getVarType(v1));
-                }
-            } else {
-                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
-                LogicalVariable v2 = vre.getVariableReference();
-                env.setVarType(v2, env2.getVarType(v2));
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getDecorList()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (p.first != null) {
-                env.setVarType(p.first, env2.getType(expr));
-            } else {
-                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
-                LogicalVariable v2 = vre.getVariableReference();
-                env.setVarType(v2, env2.getVarType(v2));
-            }
-        }
-        return env;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
deleted file mode 100644
index 94b4252..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorExtension.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.Collection;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-
-/**
- * @author rico
- */
-public interface IOperatorExtension {
-
-    void setExecutionMode(ExecutionMode mode);
-
-    boolean isMap();
-
-    public IOperatorExtension newInstance();
-
-    boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException;
-
-    void setSchema(List<LogicalVariable> schema);
-
-    IPhysicalOperator getPhysicalOperator();
-
-    void setPhysicalOperator(IPhysicalOperator physicalOperator);
-
-    ExecutionMode getExecutionMode();
-    
-    public void getUsedVariables(Collection<LogicalVariable> usedVars);
-    
-    public void getProducedVariables(Collection<LogicalVariable> producedVars);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
deleted file mode 100644
index 4e50f4f..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IOperatorSchema extends Iterable<LogicalVariable> {
-    public void addAllVariables(IOperatorSchema source);
-
-    public void addAllNewVariables(IOperatorSchema source);
-
-    public int addVariable(LogicalVariable var);
-
-    public int findVariable(LogicalVariable var);
-
-    public LogicalVariable getVariable(int index);
-
-    public int getSize();
-
-    public void clear();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
deleted file mode 100644
index c58215e..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class IndexInsertDeleteOperator extends AbstractLogicalOperator {
-
-    private final IDataSourceIndex<?, ?> dataSourceIndex;
-    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
-    // In the bulk-load case on ngram or keyword index,
-    // it contains [token, number of token] or [token].
-    // Otherwise, it contains secondary key information.
-    private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
-    private final Mutable<ILogicalExpression> filterExpr;
-    private final Kind operation;
-    private final boolean bulkload;
-    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
-
-    public IndexInsertDeleteOperator(IDataSourceIndex<?, ?> dataSourceIndex,
-            List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
-            Mutable<ILogicalExpression> filterExpr, Kind operation, boolean bulkload) {
-        this.dataSourceIndex = dataSourceIndex;
-        this.primaryKeyExprs = primaryKeyExprs;
-        this.secondaryKeyExprs = secondaryKeyExprs;
-        this.filterExpr = filterExpr;
-        this.operation = operation;
-        this.bulkload = bulkload;
-    }
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean b = false;
-        for (int i = 0; i < primaryKeyExprs.size(); i++) {
-            if (visitor.transform(primaryKeyExprs.get(i))) {
-                b = true;
-            }
-        }
-        for (int i = 0; i < secondaryKeyExprs.size(); i++) {
-            if (visitor.transform(secondaryKeyExprs.get(i))) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitIndexInsertDeleteOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.INDEX_INSERT_DELETE;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
-        return primaryKeyExprs;
-    }
-
-    public IDataSourceIndex<?, ?> getDataSourceIndex() {
-        return dataSourceIndex;
-    }
-
-    public String getIndexName() {
-        return dataSourceIndex.getId().toString();
-    }
-
-    public List<Mutable<ILogicalExpression>> getSecondaryKeyExpressions() {
-        return secondaryKeyExprs;
-    }
-
-    public Mutable<ILogicalExpression> getFilterExpression() {
-        return filterExpr;
-    }
-
-    public Kind getOperation() {
-        return operation;
-    }
-
-    public boolean isBulkload() {
-        return bulkload;
-    }
-
-    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
-        this.additionalFilteringExpressions = additionalFilteringExpressions;
-    }
-
-    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
-        return additionalFilteringExpressions;
-    }
-
-}

[10/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
deleted file mode 100644
index 0881b50..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.util;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalPropertiesVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractJoinPOperator.JoinPartitioningType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HybridHashJoinPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryHashJoinPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NLJoinPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-
-public class JoinUtils {
-
-    public static void setJoinAlgorithmAndExchangeAlgo(AbstractBinaryJoinOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        List<LogicalVariable> sideLeft = new LinkedList<LogicalVariable>();
-        List<LogicalVariable> sideRight = new LinkedList<LogicalVariable>();
-        List<LogicalVariable> varsLeft = op.getInputs().get(0).getValue().getSchema();
-        List<LogicalVariable> varsRight = op.getInputs().get(1).getValue().getSchema();
-        if (isHashJoinCondition(op.getCondition().getValue(), varsLeft, varsRight, sideLeft, sideRight)) {
-            BroadcastSide side = getBroadcastJoinSide(op.getCondition().getValue(), varsLeft, varsRight);
-            if (side == null) {
-                setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
-            } else {
-                switch (side) {
-                    case RIGHT:
-                        setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideLeft, sideRight, context);
-                        break;
-                    case LEFT:
-                        Mutable<ILogicalOperator> opRef0 = op.getInputs().get(0);
-                        Mutable<ILogicalOperator> opRef1 = op.getInputs().get(1);
-                        ILogicalOperator tmp = opRef0.getValue();
-                        opRef0.setValue(opRef1.getValue());
-                        opRef1.setValue(tmp);
-                        setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideRight, sideLeft, context);
-                        break;
-                    default:
-                        setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
-                }
-            }
-        } else {
-            setNLJoinOp(op, context);
-        }
-    }
-
-    private static void setNLJoinOp(AbstractBinaryJoinOperator op, IOptimizationContext context) {
-        op.setPhysicalOperator(new NLJoinPOperator(op.getJoinKind(), JoinPartitioningType.BROADCAST, context
-                .getPhysicalOptimizationConfig().getMaxRecordsPerFrame()));
-    }
-
-    private static void setHashJoinOp(AbstractBinaryJoinOperator op, JoinPartitioningType partitioningType,
-            List<LogicalVariable> sideLeft, List<LogicalVariable> sideRight, IOptimizationContext context)
-            throws AlgebricksException {
-        op.setPhysicalOperator(new HybridHashJoinPOperator(op.getJoinKind(), partitioningType, sideLeft, sideRight,
-                context.getPhysicalOptimizationConfig().getMaxFramesHybridHash(), context
-                        .getPhysicalOptimizationConfig().getMaxFramesLeftInputHybridHash(), context
-                        .getPhysicalOptimizationConfig().getMaxRecordsPerFrame(), context
-                        .getPhysicalOptimizationConfig().getFudgeFactor()));
-        if (partitioningType == JoinPartitioningType.BROADCAST) {
-            hybridToInMemHashJoin(op, context);
-        }
-        // op.setPhysicalOperator(new
-        // InMemoryHashJoinPOperator(op.getJoinKind(), partitioningType,
-        // sideLeft, sideRight,
-        // 1024 * 512));
-    }
-
-    private static void hybridToInMemHashJoin(AbstractBinaryJoinOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        ILogicalOperator opBuild = op.getInputs().get(1).getValue();
-        LogicalPropertiesVisitor.computeLogicalPropertiesDFS(opBuild, context);
-        ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild);
-        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch -- Logical properties for " + opBuild
-                + ": " + v + "\n");
-        if (v != null) {
-            int size2 = v.getMaxOutputFrames();
-            HybridHashJoinPOperator hhj = (HybridHashJoinPOperator) op.getPhysicalOperator();
-            if (size2 > 0 && size2 * hhj.getFudgeFactor() <= hhj.getMemSizeInFrames()) {
-                AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch " + opBuild
-                        + " fits in memory\n");
-                // maintains the local properties on the probe side
-                op.setPhysicalOperator(new InMemoryHashJoinPOperator(hhj.getKind(), hhj.getPartitioningType(), hhj
-                        .getKeysLeftBranch(), hhj.getKeysRightBranch(), v.getNumberOfTuples() * 2));
-            }
-        }
-
-    }
-
-    private static boolean isHashJoinCondition(ILogicalExpression e, Collection<LogicalVariable> inLeftAll,
-            Collection<LogicalVariable> inRightAll, Collection<LogicalVariable> outLeftFields,
-            Collection<LogicalVariable> outRightFields) {
-        switch (e.getExpressionTag()) {
-            case FUNCTION_CALL: {
-                AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
-                FunctionIdentifier fi = fexp.getFunctionIdentifier();
-                if (fi.equals(AlgebricksBuiltinFunctions.AND)) {
-                    for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
-                        if (!isHashJoinCondition(a.getValue(), inLeftAll, inRightAll, outLeftFields,
-                                outRightFields)) {
-                            return false;
-                        }
-                    }
-                    return true;
-                } else {
-                    ComparisonKind ck = AlgebricksBuiltinFunctions.getComparisonType(fi);
-                    if (ck != ComparisonKind.EQ) {
-                        return false;
-                    }
-                    ILogicalExpression opLeft = fexp.getArguments().get(0).getValue();
-                    ILogicalExpression opRight = fexp.getArguments().get(1).getValue();
-                    if (opLeft.getExpressionTag() != LogicalExpressionTag.VARIABLE
-                            || opRight.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                        return false;
-                    }
-                    LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
-                    if (inLeftAll.contains(var1) && !outLeftFields.contains(var1)) {
-                        outLeftFields.add(var1);
-                    } else if (inRightAll.contains(var1) && !outRightFields.contains(var1)) {
-                        outRightFields.add(var1);
-                    } else {
-                        return false;
-                    }
-                    LogicalVariable var2 = ((VariableReferenceExpression) opRight).getVariableReference();
-                    if (inLeftAll.contains(var2) && !outLeftFields.contains(var2)) {
-                        outLeftFields.add(var2);
-                    } else if (inRightAll.contains(var2) && !outRightFields.contains(var2)) {
-                        outRightFields.add(var2);
-                    } else {
-                        return false;
-                    }
-                    return true;
-                }
-            }
-            default: {
-                return false;
-            }
-        }
-    }
-
-    private static BroadcastSide getBroadcastJoinSide(ILogicalExpression e, List<LogicalVariable> varsLeft,
-            List<LogicalVariable> varsRight) {
-        if (e.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return null;
-        }
-        AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
-        IExpressionAnnotation ann = fexp.getAnnotations().get(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY);
-        if (ann == null) {
-            return null;
-        }
-        BroadcastSide side = (BroadcastSide) ann.getObject();
-        if (side == null) {
-            return null;
-        }
-        int i;
-        switch (side) {
-            case LEFT:
-                i = 0;
-                break;
-            case RIGHT:
-                i = 1;
-                break;
-            default:
-                return null;
-        }
-        ArrayList<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-        fexp.getArguments().get(i).getValue().getUsedVariables(vars);
-        if (varsLeft.containsAll(vars)) {
-            return BroadcastSide.LEFT;
-        } else if (varsRight.containsAll(vars)) {
-            return BroadcastSide.RIGHT;
-        } else {
-            return null;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
deleted file mode 100644
index ecacc05..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.util;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-
-public class PhysicalOptimizationsUtil {
-
-    public static void computeFDsAndEquivalenceClasses(AbstractLogicalOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        FDsAndEquivClassesVisitor visitor = new FDsAndEquivClassesVisitor();
-        Set<ILogicalOperator> visitSet = new HashSet<ILogicalOperator>();
-        computeFDsAndEqClassesWithVisitorRec(op, ctx, visitor, visitSet);
-    }
-
-    private static void computeFDsAndEqClassesWithVisitorRec(AbstractLogicalOperator op, IOptimizationContext ctx,
-            FDsAndEquivClassesVisitor visitor, Set<ILogicalOperator> visitSet) throws AlgebricksException {
-        visitSet.add(op);
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) i.getValue(), ctx, visitor, visitSet);
-        }
-        if (op.hasNestedPlans()) {
-            for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    AbstractLogicalOperator rootOp = (AbstractLogicalOperator) r.getValue();
-                    computeFDsAndEqClassesWithVisitorRec(rootOp, ctx, visitor, visitSet);
-                }
-            }
-        }
-        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
-            ILogicalOperator source = nts.getDataSourceReference().getValue().getInputs().get(0).getValue();
-            if (!visitSet.contains(source)) {
-                computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) source, ctx, visitor, visitSet);
-            }
-        }
-        op.accept(visitor, ctx);
-        if (AlgebricksConfig.DEBUG) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine("--> op. type = " + op.getOperatorTag() + "\n"
-                    + "    equiv. classes = " + ctx.getEquivalenceClassMap(op) + "\n" + "    FDs = "
-                    + ctx.getFDList(op) + "\n");
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
new file mode 100644
index 0000000..f897791
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
+
+public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    protected boolean descOrSelfIsScanOrJoin(AbstractLogicalOperator op2) {
+        LogicalOperatorTag t = op2.getOperatorTag();
+        if (isScanOrJoin(t)) {
+            return true;
+        }
+        if (op2.getInputs().size() != 1) {
+            return false;
+        }
+        AbstractLogicalOperator alo = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+        if (descOrSelfIsScanOrJoin(alo)) {
+            return true;
+        }
+        return false;
+    }
+
+    protected boolean isScanOrJoin(LogicalOperatorTag t) {
+        if (t == LogicalOperatorTag.DATASOURCESCAN || t == LogicalOperatorTag.INNERJOIN
+                || t == LogicalOperatorTag.UNNEST || t == LogicalOperatorTag.UNNEST_MAP
+                || t == LogicalOperatorTag.LEFTOUTERJOIN) {
+            return true;
+        }
+        return false;
+    }
+
+    protected Set<LogicalVariable> computeGbyVarsUsingPksOnly(Set<LogicalVariable> varSet, AbstractLogicalOperator op,
+            IOptimizationContext context) throws AlgebricksException {
+        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
+        List<FunctionalDependency> fdList = context.getFDList(op);
+        if (fdList == null) {
+            return null;
+        }
+        // check if any of the FDs is a key
+        for (FunctionalDependency fd : fdList) {
+            if (fd.getTail().containsAll(varSet)) {
+                return new HashSet<LogicalVariable>(fd.getHead());
+            }
+        }
+        return null;
+    }
+
+    protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
+        for (LogicalVariable ov : vars) {
+            LogicalVariable newVar = context.newVar();
+            ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
+            outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
+                    new MutableObject<ILogicalExpression>(varExpr)));
+            for (ILogicalPlan p : g.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
+                            context);
+                }
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
new file mode 100644
index 0000000..882db41
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public abstract class AbstractExtractExprRule implements IAlgebraicRewriteRule {
+
+    protected LogicalVariable extractExprIntoAssignOpRef(ILogicalExpression gExpr, Mutable<ILogicalOperator> opRef2,
+            IOptimizationContext context) throws AlgebricksException {
+        LogicalVariable v = context.newVar();
+        AssignOperator a = new AssignOperator(v, new MutableObject<ILogicalExpression>(gExpr));
+        a.getInputs().add(new MutableObject<ILogicalOperator>(opRef2.getValue()));
+        opRef2.setValue(a);
+        if (gExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+            context.addNotToBeInlinedVar(v);
+        }
+        context.computeAndSetTypeEnvironmentForOperator(a);
+        return v;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
new file mode 100644
index 0000000..67b2d0c
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public abstract class AbstractIntroduceCombinerRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    /**
+     * Replace the original aggregate functions with their corresponding global aggregate function.
+     */
+    protected void replaceOriginalAggFuncs(Set<SimilarAggregatesInfo> toReplaceSet) {
+        for (SimilarAggregatesInfo sai : toReplaceSet) {
+            for (AggregateExprInfo aei : sai.simAggs) {
+                AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getValue();
+                afce.setFunctionInfo(aei.newFunInfo);
+                afce.getArguments().clear();
+                afce.getArguments().add(new MutableObject<ILogicalExpression>(sai.stepOneResult));
+            }
+        }
+    }
+
+    protected Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg,
+            GroupByOperator newGbyOp, Set<SimilarAggregatesInfo> toReplaceSet, IOptimizationContext context)
+            throws AlgebricksException {
+
+        ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
+        ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
+
+        List<LogicalVariable> initVars = initAgg.getVariables();
+        List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
+        int numExprs = initVars.size();
+
+        // First make sure that all agg funcs are two step, otherwise we cannot use local aggs.
+        for (int i = 0; i < numExprs; i++) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
+            if (!aggFun.isTwoStep()) {
+                return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
+            }
+        }
+
+        boolean haveAggToReplace = false;
+        for (int i = 0; i < numExprs; i++) {
+            Mutable<ILogicalExpression> expRef = initExprs.get(i);
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
+            IFunctionInfo fi1 = aggFun.getStepOneAggregate();
+            // Clone the aggregate's args.
+            List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun
+                    .getArguments().size());
+            for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
+                newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
+            }
+            IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
+
+            SimilarAggregatesInfo inf = new SimilarAggregatesInfo();
+            LogicalVariable newAggVar = context.newVar();
+            pushedVars.add(newAggVar);
+            inf.stepOneResult = new VariableReferenceExpression(newAggVar);
+            inf.simAggs = new ArrayList<AggregateExprInfo>();
+            toReplaceSet.add(inf);
+            AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
+            pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
+            AggregateExprInfo aei = new AggregateExprInfo();
+            aei.aggExprRef = expRef;
+            aei.newFunInfo = fi2;
+            inf.simAggs.add(aei);
+            haveAggToReplace = true;
+        }
+
+        if (!pushedVars.isEmpty()) {
+            AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
+            pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
+            // If newGbyOp is null, then we optimizing an aggregate without group by.
+            if (newGbyOp != null) {
+                // Cut and paste nested input pipelines of initAgg to pushedAgg's input
+                Mutable<ILogicalOperator> inputRef = initAgg.getInputs().get(0);
+                Mutable<ILogicalOperator> bottomRef = inputRef;
+                while (bottomRef.getValue().getInputs().size() > 0) {
+                    bottomRef = bottomRef.getValue().getInputs().get(0);
+                }
+                ILogicalOperator oldNts = bottomRef.getValue();
+                initAgg.getInputs().clear();
+                initAgg.getInputs().add(new MutableObject<ILogicalOperator>(oldNts));
+
+                // Hook up the nested aggregate op with the outer group by.
+                NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(
+                        newGbyOp));
+                nts.setExecutionMode(ExecutionMode.LOCAL);
+                bottomRef.setValue(nts);
+                pushedAgg.getInputs().add(inputRef);
+            } else {
+                // The local aggregate operator is fed by the input of the original aggregate operator.
+                pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(initAgg.getInputs().get(0).getValue()));
+                // Reintroduce assign op for the global agg partitioning var.
+                initAgg.getInputs().get(0).setValue(pushedAgg);
+                pushedAgg.setGlobal(false);
+                context.computeAndSetTypeEnvironmentForOperator(pushedAgg);
+            }
+            return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
+        } else {
+            return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
+        }
+    }
+
+    protected class SimilarAggregatesInfo {
+        ILogicalExpression stepOneResult;
+        List<AggregateExprInfo> simAggs;
+    }
+
+    protected class AggregateExprInfo {
+        Mutable<ILogicalExpression> aggExprRef;
+        IFunctionInfo newFunInfo;
+    }
+
+    protected class BookkeepingInfo {
+        Map<GroupByOperator, List<LogicalVariable>> modifyGbyMap = new HashMap<GroupByOperator, List<LogicalVariable>>();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java
new file mode 100644
index 0000000..b60c80d
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java
@@ -0,0 +1,340 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+
+public abstract class AbstractIntroduceGroupByCombinerRule extends AbstractIntroduceCombinerRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (context.checkIfInDontApplySet(this, op)) {
+            return false;
+        }
+        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        GroupByOperator gbyOp = (GroupByOperator) op;
+        if (gbyOp.getExecutionMode() != ExecutionMode.PARTITIONED) {
+            return false;
+        }
+
+        BookkeepingInfo bi = new BookkeepingInfo();
+        GroupByOperator newGbyOp = opToPush(gbyOp, bi, context);
+        if (newGbyOp == null) {
+            return false;
+        }
+
+        Set<LogicalVariable> newGbyLiveVars = new ListSet<LogicalVariable>();
+        VariableUtilities.getLiveVariables(newGbyOp, newGbyLiveVars);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
+            List<LogicalVariable> usedDecorVars = new ArrayList<LogicalVariable>();
+            // p.second.getValue() should always return a VariableReferenceExpression, hence
+            // usedDecorVars should always contain only one variable.
+            p.second.getValue().getUsedVariables(usedDecorVars);
+            if (!newGbyLiveVars.contains(usedDecorVars.get(0))) {
+                // Let the left-hand side of gbyOp's decoration expressions populated through the combiner group-by without
+                // any intermediate assignment.
+                newGbyOp.addDecorExpression(null, p.second.getValue());
+            }
+        }
+        newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
+        Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
+        newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v);
+
+        Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY);
+        newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2);
+
+        List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(newGbyOp, propagatedVars);
+
+        Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars);
+
+        for (LogicalVariable var : freeVars) {
+            if (!propagatedVars.contains(var)) {
+                LogicalVariable newDecorVar = context.newVar();
+                newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
+                VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
+                        newDecorVar, context);
+            }
+        }
+
+        Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+        opRef3.setValue(newGbyOp);
+        typeGby(newGbyOp, context);
+        typeGby(gbyOp, context);
+        context.addToDontApplySet(this, op);
+        return true;
+    }
+
+    private void typeGby(AbstractOperatorWithNestedPlans op, IOptimizationContext context) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            OperatorPropertiesUtil.typePlan(p, context);
+        }
+        context.computeAndSetTypeEnvironmentForOperator(op);
+    }
+
+    private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context)
+            throws AlgebricksException {
+        // Hook up input to new group-by.
+        Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+        ILogicalOperator op3 = opRef3.getValue();
+        GroupByOperator newGbyOp = new GroupByOperator();
+        newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
+        // Copy annotations.
+        Map<String, Object> annotations = newGbyOp.getAnnotations();
+        annotations.putAll(gbyOp.getAnnotations());
+
+        List<LogicalVariable> gbyVars = gbyOp.getGbyVarList();
+        for (ILogicalPlan p : gbyOp.getNestedPlans()) {
+            Pair<Boolean, ILogicalPlan> bip = tryToPushSubplan(p, gbyOp, newGbyOp, bi, gbyVars, context);
+            if (!bip.first) {
+                // For now, if we cannot push everything, give up.
+                return null;
+            }
+            ILogicalPlan pushedSubplan = bip.second;
+            if (pushedSubplan != null) {
+                newGbyOp.getNestedPlans().add(pushedSubplan);
+            }
+        }
+
+        ArrayList<LogicalVariable> newOpGbyList = new ArrayList<LogicalVariable>();
+        ArrayList<LogicalVariable> replGbyList = new ArrayList<LogicalVariable>();
+        // Find maximal sequence of variable.
+        for (Map.Entry<GroupByOperator, List<LogicalVariable>> e : bi.modifyGbyMap.entrySet()) {
+            List<LogicalVariable> varList = e.getValue();
+            boolean see1 = true;
+            int sz1 = newOpGbyList.size();
+            int i = 0;
+            for (LogicalVariable v : varList) {
+                if (see1) {
+                    if (i < sz1) {
+                        LogicalVariable v2 = newOpGbyList.get(i);
+                        if (v != v2) {
+                            // cannot linearize
+                            return null;
+                        }
+                    } else {
+                        see1 = false;
+                        newOpGbyList.add(v);
+                        replGbyList.add(context.newVar());
+                    }
+                    i++;
+                } else {
+                    newOpGbyList.add(v);
+                    replGbyList.add(context.newVar());
+                }
+            }
+        }
+        // set the vars in the new op
+        int n = newOpGbyList.size();
+        for (int i = 0; i < n; i++) {
+            newGbyOp.addGbyExpression(replGbyList.get(i), new VariableReferenceExpression(newOpGbyList.get(i)));
+            VariableUtilities.substituteVariables(gbyOp, newOpGbyList.get(i), replGbyList.get(i), false, context);
+        }
+        return newGbyOp;
+    }
+
+    private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan nestedPlan, GroupByOperator oldGbyOp,
+            GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context)
+            throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
+        Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
+        for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
+            if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, pushedRoots, toReplaceSet)) {
+                // For now, if we cannot push everything, give up.
+                return new Pair<Boolean, ILogicalPlan>(false, null);
+            }
+        }
+        if (pushedRoots.isEmpty()) {
+            return new Pair<Boolean, ILogicalPlan>(true, null);
+        } else {
+            // Replaces the aggregation expressions in the original group-by op with new ones.
+            ILogicalPlan newPlan = new ALogicalPlanImpl(pushedRoots);
+            ILogicalPlan plan = fingIdenticalPlan(newGbyOp, newPlan);
+            replaceOriginalAggFuncs(toReplaceSet);
+            if (plan == null) {
+                return new Pair<Boolean, ILogicalPlan>(true, newPlan);
+            } else {
+                // Does not add a nested subplan to newGbyOp if there already exists an isomorphic plan.
+                Set<LogicalVariable> originalVars = new ListSet<LogicalVariable>();
+                Set<LogicalVariable> newVars = new ListSet<LogicalVariable>();
+                for (Mutable<ILogicalOperator> rootRef : pushedRoots) {
+                    VariableUtilities.getProducedVariables(rootRef.getValue(), originalVars);
+                }
+                for (Mutable<ILogicalOperator> rootRef : plan.getRoots()) {
+                    VariableUtilities.getProducedVariables(rootRef.getValue(), newVars);
+                }
+
+                // Replaces variable exprs referring to the variables produced by newPlan by 
+                // those produced by plan.
+                Iterator<LogicalVariable> originalVarIter = originalVars.iterator();
+                Iterator<LogicalVariable> newVarIter = newVars.iterator();
+                while (originalVarIter.hasNext()) {
+                    LogicalVariable originalVar = originalVarIter.next();
+                    LogicalVariable newVar = newVarIter.next();
+                    for (SimilarAggregatesInfo sai : toReplaceSet) {
+                        for (AggregateExprInfo aei : sai.simAggs) {
+                            ILogicalExpression afce = aei.aggExprRef.getValue();
+                            afce.substituteVar(originalVar, newVar);
+                        }
+                    }
+                }
+                return new Pair<Boolean, ILogicalPlan>(true, null);
+            }
+        }
+    }
+
+    private ILogicalPlan fingIdenticalPlan(GroupByOperator newGbyOp, ILogicalPlan plan) throws AlgebricksException {
+        for (ILogicalPlan nestedPlan : newGbyOp.getNestedPlans()) {
+            if (IsomorphismUtilities.isOperatorIsomorphicPlan(plan, nestedPlan)) {
+                return nestedPlan;
+            }
+        }
+        return null;
+    }
+
+    private boolean tryToPushRoot(Mutable<ILogicalOperator> root, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
+            BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context,
+            List<Mutable<ILogicalOperator>> toPushAccumulate, Set<SimilarAggregatesInfo> toReplaceSet)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) root.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+        // Finds nested group-by if any.
+        AbstractLogicalOperator op3 = op2;
+        while (op3.getOperatorTag() != LogicalOperatorTag.GROUP && op3.getInputs().size() == 1) {
+            op3 = (AbstractLogicalOperator) op3.getInputs().get(0).getValue();
+        }
+
+        if (op3.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            AggregateOperator initAgg = (AggregateOperator) op1;
+            Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, toReplaceSet, context);
+            if (!pOpRef.first) {
+                return false;
+            }
+            Mutable<ILogicalOperator> opRef = pOpRef.second;
+            if (opRef != null) {
+                toPushAccumulate.add(opRef);
+            }
+            bi.modifyGbyMap.put(oldGbyOp, gbyVars);
+            return true;
+        } else {
+            GroupByOperator nestedGby = (GroupByOperator) op3;
+            List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
+            List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
+            concatGbyVars.addAll(gbyVars2);
+            for (ILogicalPlan p : nestedGby.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
+                    if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate,
+                            toReplaceSet)) {
+                        return false;
+                    }
+                }
+            }
+
+            /***
+             * Push the nested pipeline which provides the input to the nested group operator into newGbyOp (the combined gby op).
+             * The change is to fix asterixdb issue 782.
+             */
+            Mutable<ILogicalOperator> nestedGbyInputRef = nestedGby.getInputs().get(0);
+            Mutable<ILogicalOperator> startOfPipelineRef = nestedGbyInputRef;
+            if (startOfPipelineRef.getValue().getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                return true;
+            }
+
+            // move down the nested pipeline to find the start of the pipeline right upon the nested-tuple-source
+            boolean hasIsNullFunction = OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) startOfPipelineRef
+                    .getValue());
+            while (startOfPipelineRef.getValue().getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                startOfPipelineRef = startOfPipelineRef.getValue().getInputs().get(0);
+                hasIsNullFunction = OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) startOfPipelineRef
+                        .getValue());
+            }
+            //keep the old nested-tuple-source
+            Mutable<ILogicalOperator> oldNts = startOfPipelineRef.getValue().getInputs().get(0);
+
+            //move down the nested op in the new gby operator
+            Mutable<ILogicalOperator> newGbyNestedOpRef = toPushAccumulate.get(0);
+            while (newGbyNestedOpRef.getValue().getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                newGbyNestedOpRef = newGbyNestedOpRef.getValue().getInputs().get(0);
+            }
+
+            //insert the pipeline before nested gby into the new (combiner) gby's nested plan on top of the nested-tuple-source
+            startOfPipelineRef.getValue().getInputs().set(0, newGbyNestedOpRef.getValue().getInputs().get(0));
+            newGbyNestedOpRef.getValue().getInputs().set(0, nestedGbyInputRef);
+
+            //in the old gby operator, remove the nested pipeline since it is already pushed to the combiner gby
+            nestedGby.getInputs().set(0, oldNts);
+            List<LogicalVariable> aggProducedVars = new ArrayList<LogicalVariable>();
+            VariableUtilities.getProducedVariables(toPushAccumulate.get(0).getValue(), aggProducedVars);
+
+            if (hasIsNullFunction && aggProducedVars.size() != 0) {
+                // if the old nested pipeline contains a not-null-check, we need to convert it to a not-system-null-check in the non-local gby
+                processNullTest(context, nestedGby, aggProducedVars);
+            }
+
+            return true;
+        }
+    }
+
+    /**
+     * Deal with the case where the nested plan in the combiner gby operator has a null-test before invoking aggregation functions.
+     * 
+     * @param context
+     *            The optimization context.
+     * @param nestedGby
+     *            The nested gby operator in the global gby operator's subplan.
+     * @param firstAggVar
+     *            The first aggregation variable produced by the combiner gby.
+     */
+    protected abstract void processNullTest(IOptimizationContext context, GroupByOperator nestedGby,
+            List<LogicalVariable> aggregateVarsProducedByCombiner);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
new file mode 100644
index 0000000..53ca903
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class BreakSelectIntoConjunctsRule implements IAlgebraicRewriteRule {
+
+    private List<Mutable<ILogicalExpression>> conjs = new ArrayList<Mutable<ILogicalExpression>>();
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+        SelectOperator select = (SelectOperator) op;
+
+        ILogicalExpression cond = select.getCondition().getValue();
+
+        conjs.clear();
+        if (!cond.splitIntoConjuncts(conjs)) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> childOfSelect = select.getInputs().get(0);
+        boolean fst = true;
+        ILogicalOperator botOp = select;
+        ILogicalExpression firstExpr = null;
+        for (Mutable<ILogicalExpression> eRef : conjs) {
+            ILogicalExpression e = eRef.getValue();
+            if (fst) {
+                fst = false;
+                firstExpr = e;
+            } else {
+                SelectOperator newSelect = new SelectOperator(new MutableObject<ILogicalExpression>(e),
+                        select.getRetainNull(), select.getNullPlaceholderVariable());
+                List<Mutable<ILogicalOperator>> botInpList = botOp.getInputs();
+                botInpList.clear();
+                botInpList.add(new MutableObject<ILogicalOperator>(newSelect));
+                context.computeAndSetTypeEnvironmentForOperator(botOp);
+                botOp = newSelect;
+            }
+        }
+        botOp.getInputs().add(childOfSelect);
+        select.getCondition().setValue(firstExpr);
+        context.computeAndSetTypeEnvironmentForOperator(botOp);
+        context.computeAndSetTypeEnvironmentForOperator(select);
+
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
new file mode 100644
index 0000000..73fd966
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
@@ -0,0 +1,138 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ComplexJoinInferenceRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
+        if (!(op instanceof AbstractScanOperator)) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op2;
+
+        Mutable<ILogicalOperator> opRef3 = subplan.getInputs().get(0);
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
+
+        if (op3.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
+                || op3.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            return false;
+        }
+
+        if (subplanHasFreeVariables(subplan)) {
+            return false;
+        }
+
+        HashSet<LogicalVariable> varsUsedInUnnest = new HashSet<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op, varsUsedInUnnest);
+
+        HashSet<LogicalVariable> producedInSubplan = new HashSet<LogicalVariable>();
+        VariableUtilities.getProducedVariables(subplan, producedInSubplan);
+
+        if (!producedInSubplan.containsAll(varsUsedInUnnest)) {
+            return false;
+        }
+
+        ntsToEtsInSubplan(subplan, context);
+        cleanupJoins(subplan);
+        InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
+        join.getInputs().add(opRef3);
+        opRef2.setValue(OperatorManipulationUtil.eliminateSingleSubplanOverEts(subplan));
+        join.getInputs().add(new MutableObject<ILogicalOperator>(op));
+        opRef.setValue(join);
+        context.computeAndSetTypeEnvironmentForOperator(join);
+        return true;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    private static void cleanupJoins(SubplanOperator s) {
+        for (ILogicalPlan p : s.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                cleanupJoins(r);
+            }
+        }
+    }
+
+    /** clean up joins that have one input branch that is empty tuple source */
+    private static void cleanupJoins(Mutable<ILogicalOperator> opRef) {
+        if (opRef.getValue() instanceof AbstractBinaryJoinOperator) {
+            for (Mutable<ILogicalOperator> inputRef : opRef.getValue().getInputs()) {
+                if (inputRef.getValue().getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+                    opRef.getValue().getInputs().remove(inputRef);
+                    opRef.setValue(opRef.getValue().getInputs().get(0).getValue());
+                    break;
+                }
+            }
+        }
+        for (Mutable<ILogicalOperator> inputRef : opRef.getValue().getInputs()) {
+            cleanupJoins(inputRef);
+        }
+    }
+
+    private static void ntsToEtsInSubplan(SubplanOperator s, IOptimizationContext context) throws AlgebricksException {
+        for (ILogicalPlan p : s.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.ntsToEts(r, context);
+            }
+        }
+    }
+
+    private static boolean subplanHasFreeVariables(SubplanOperator s) throws AlgebricksException {
+        for (ILogicalPlan p : s.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                if (OperatorPropertiesUtil.hasFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue())) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
new file mode 100644
index 0000000..910a8f9
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
@@ -0,0 +1,314 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Complex rewrite rule for producing joins from unnests.
+ * This rule is limited to creating left-deep trees.
+ */
+public class ComplexUnnestToProductRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN
+                && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return false;
+        }
+
+        //stop rewriting if the operators originates from a nested tuple source
+        if (insideSubplan(opRef)) {
+            return false;
+        }
+
+        // We may pull selects above the join we create in order to eliminate possible dependencies between
+        // the outer and inner input plans of the join.
+        List<ILogicalOperator> topSelects = new ArrayList<ILogicalOperator>();
+
+        // Keep track of the operators and used variables participating in the inner input plan.
+        HashSet<LogicalVariable> innerUsedVars = new HashSet<LogicalVariable>();
+        List<ILogicalOperator> innerOps = new ArrayList<ILogicalOperator>();
+        HashSet<LogicalVariable> outerUsedVars = new HashSet<LogicalVariable>();
+        List<ILogicalOperator> outerOps = new ArrayList<ILogicalOperator>();
+        innerOps.add(op);
+        VariableUtilities.getUsedVariables(op, innerUsedVars);
+
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+        // Find an unnest or join and partition the plan between the first unnest and that operator into independent parts.
+        if (!findPlanPartition(op2, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, false)) {
+            // We could not find an unnest or join.
+            return false;
+        }
+        // The last operator must be an unnest or join.
+        AbstractLogicalOperator unnestOrJoin = (AbstractLogicalOperator) outerOps.get(outerOps.size() - 1);
+
+        ILogicalOperator outerRoot = null;
+        ILogicalOperator innerRoot = null;
+        EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
+        // If we found a join, simply use it as the outer root.
+        if (unnestOrJoin.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+                && unnestOrJoin.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+            // We've found a second unnest. First, sanity check that the unnest does not produce any vars that are used by the plan above (until the first unnest).
+            List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+            VariableUtilities.getProducedVariables(unnestOrJoin, producedVars);
+            for (LogicalVariable producedVar : producedVars) {
+                if (innerUsedVars.contains(producedVar)) {
+                    return false;
+                }
+            }
+            // Continue finding a partitioning of the plan such that the inner and outer partitions are independent, in order to feed a join.
+            // Now, we look below the second unnest or join.
+            VariableUtilities.getUsedVariables(unnestOrJoin, outerUsedVars);
+            AbstractLogicalOperator unnestChild = (AbstractLogicalOperator) unnestOrJoin.getInputs().get(0).getValue();
+            if (!findPlanPartition(unnestChild, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, true)) {
+                // We could not find a suitable partitioning.
+                return false;
+            }
+        }
+        innerRoot = buildOperatorChain(innerOps, ets, context);
+        context.computeAndSetTypeEnvironmentForOperator(innerRoot);
+        outerRoot = buildOperatorChain(outerOps, null, context);
+        context.computeAndSetTypeEnvironmentForOperator(outerRoot);
+
+        InnerJoinOperator product = new InnerJoinOperator(
+                new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
+        // Outer branch.
+        product.getInputs().add(new MutableObject<ILogicalOperator>(outerRoot));
+        // Inner branch.
+        product.getInputs().add(new MutableObject<ILogicalOperator>(innerRoot));
+        context.computeAndSetTypeEnvironmentForOperator(product);
+        // Put the selects on top of the join.
+        ILogicalOperator topOp = product;
+        if (!topSelects.isEmpty()) {
+            topOp = buildOperatorChain(topSelects, product, context);
+        }
+        // Plug the selects + product in the plan.
+        opRef.setValue(topOp);
+        context.computeAndSetTypeEnvironmentForOperator(topOp);
+        return true;
+    }
+
+    private ILogicalOperator buildOperatorChain(List<ILogicalOperator> ops, ILogicalOperator bottomOp,
+            IOptimizationContext context) throws AlgebricksException {
+        ILogicalOperator root = ops.get(0);
+        ILogicalOperator prevOp = root;
+        for (int i = 1; i < ops.size(); i++) {
+            ILogicalOperator inputOp = ops.get(i);
+            prevOp.getInputs().clear();
+            prevOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
+            prevOp = inputOp;
+        }
+        if (bottomOp != null) {
+            context.computeAndSetTypeEnvironmentForOperator(bottomOp);
+            prevOp.getInputs().clear();
+            prevOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomOp));
+        }
+        return root;
+    }
+
+    private boolean findPlanPartition(AbstractLogicalOperator op, HashSet<LogicalVariable> innerUsedVars,
+            HashSet<LogicalVariable> outerUsedVars, List<ILogicalOperator> innerOps, List<ILogicalOperator> outerOps,
+            List<ILogicalOperator> topSelects, boolean belowSecondUnnest) throws AlgebricksException {
+        if (belowSecondUnnest && innerUsedVars.isEmpty()) {
+            // Trivially joinable.
+            return true;
+        }
+        if (!belowSecondUnnest) {
+            // Bail on the following operators.
+            switch (op.getOperatorTag()) {
+                case AGGREGATE:
+                case SUBPLAN:
+                case GROUP:
+                case UNNEST_MAP:
+                    return false;
+            }
+        }
+        switch (op.getOperatorTag()) {
+            case UNNEST:
+            case DATASOURCESCAN: {
+                // We may have reached this state by descending through a subplan.
+                outerOps.add(op);
+                return true;
+            }
+            case INNERJOIN:
+            case LEFTOUTERJOIN: {
+                // Make sure that no variables that are live under this join are needed by the inner.
+                List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+                VariableUtilities.getLiveVariables(op, liveVars);
+                for (LogicalVariable liveVar : liveVars) {
+                    if (innerUsedVars.contains(liveVar)) {
+                        return false;
+                    }
+                }
+                outerOps.add(op);
+                return true;
+            }
+            case SELECT: {
+                // Remember this select to pulling it above the join.
+                if (innerUsedVars.isEmpty()) {
+                    outerOps.add(op);
+                } else {
+                    topSelects.add(op);
+                }
+                break;
+            }
+            case PROJECT: {
+                // Throw away projects from the plan since we are pulling selects up.
+                break;
+            }
+            case EMPTYTUPLESOURCE:
+            case NESTEDTUPLESOURCE: {
+                if (belowSecondUnnest) {
+                    // We have successfully partitioned the plan into independent parts to be plugged into the join.
+                    return true;
+                } else {
+                    // We could not find a second unnest or a join.
+                    return false;
+                }
+            }
+            default: {
+                // The inner is trivially independent.
+                if (!belowSecondUnnest && innerUsedVars.isEmpty()) {
+                    outerOps.add(op);
+                    break;
+                }
+
+                // Examine produced vars to determine which partition uses them.
+                List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+                VariableUtilities.getProducedVariables(op, producedVars);
+                int outerMatches = 0;
+                int innerMatches = 0;
+                for (LogicalVariable producedVar : producedVars) {
+                    if (outerUsedVars.contains(producedVar)) {
+                        outerMatches++;
+                    }
+                    if (innerUsedVars.contains(producedVar)) {
+                        innerMatches++;
+                    }
+                }
+
+                HashSet<LogicalVariable> targetUsedVars = null;
+                if (outerMatches == producedVars.size() && !producedVars.isEmpty()) {
+                    // All produced vars used by outer partition.
+                    outerOps.add(op);
+                    targetUsedVars = outerUsedVars;
+                }
+                if (innerMatches == producedVars.size() && !producedVars.isEmpty()) {
+                    // All produced vars used by inner partition.
+                    innerOps.add(op);
+                    targetUsedVars = innerUsedVars;
+                }
+                if (innerMatches == 0 && outerMatches == 0) {
+                    // Op produces variables that are not used in the part of the plan we've seen (or it doesn't produce any vars).
+                    // Try to figure out where it belongs by analyzing the used variables.
+                    List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getUsedVariables(op, usedVars);
+                    for (LogicalVariable usedVar : usedVars) {
+                        boolean canBreak = false;
+                        if (outerUsedVars.contains(usedVar)) {
+                            outerOps.add(op);
+                            targetUsedVars = outerUsedVars;
+                            canBreak = true;
+                        }
+                        if (innerUsedVars.contains(usedVar)) {
+                            innerOps.add(op);
+                            targetUsedVars = innerUsedVars;
+                            canBreak = true;
+                        }
+                        if (canBreak) {
+                            break;
+                        }
+                    }
+                    // TODO: For now we bail here, but we could remember such ops and determine their target partition at a later point.
+                    if (targetUsedVars == null) {
+                        return false;
+                    }
+                } else if (innerMatches != 0 && outerMatches != 0) {
+                    // The current operator produces variables that are used by both partitions, so the inner and outer are not independent and, therefore, we cannot create a join.
+                    // TODO: We may still be able to split the operator to create a viable partitioning.
+                    return false;
+                }
+                // Update used variables of partition that op belongs to.
+                if (op.hasNestedPlans() && op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+                    AbstractOperatorWithNestedPlans opWithNestedPlans = (AbstractOperatorWithNestedPlans) op;
+                    opWithNestedPlans.getUsedVariablesExceptNestedPlans(targetUsedVars);
+                } else {
+                    VariableUtilities.getUsedVariables(op, targetUsedVars);
+                }
+                break;
+            }
+        }
+        if (!op.hasInputs()) {
+            if (!belowSecondUnnest) {
+                // We could not find a second unnest or a join.
+                return false;
+            } else {
+                // We have successfully partitioned the plan into independent parts to be plugged into the join.
+                return true;
+            }
+        }
+        return findPlanPartition((AbstractLogicalOperator) op.getInputs().get(0).getValue(), innerUsedVars,
+                outerUsedVars, innerOps, outerOps, topSelects, belowSecondUnnest);
+    }
+
+    /**
+     * check whether the operator is inside a sub-plan
+     * 
+     * @param nestedRootRef
+     * @return true-if it is; false otherwise.
+     */
+    private boolean insideSubplan(Mutable<ILogicalOperator> nestedRootRef) {
+        AbstractLogicalOperator nestedRoot = (AbstractLogicalOperator) nestedRootRef.getValue();
+        if (nestedRoot.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            return true;
+        }
+        List<Mutable<ILogicalOperator>> inputs = nestedRoot.getInputs();
+        for (Mutable<ILogicalOperator> input : inputs) {
+            if (insideSubplan(input)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
new file mode 100644
index 0000000..4da332c
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator assign1 = (AssignOperator) op;
+
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) assign1.getInputs().get(0).getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator assign2 = (AssignOperator) op2;
+
+        HashSet<LogicalVariable> used1 = new HashSet<LogicalVariable>();
+        VariableUtilities.getUsedVariables(assign1, used1);
+        for (LogicalVariable v2 : assign2.getVariables()) {
+            if (used1.contains(v2)) {
+                return false;
+            }
+        }
+
+        assign1.getVariables().addAll(assign2.getVariables());
+        assign1.getExpressions().addAll(assign2.getExpressions());
+
+        Mutable<ILogicalOperator> botOpRef = assign2.getInputs().get(0);
+        List<Mutable<ILogicalOperator>> asgnInpList = assign1.getInputs();
+        asgnInpList.clear();
+        asgnInpList.add(botOpRef);
+        context.computeAndSetTypeEnvironmentForOperator(assign1);
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
new file mode 100644
index 0000000..e57bd3c
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Matches the following operator pattern:
+ * (select) <-- ((assign)* <-- (select)*)+
+ * 
+ * Consolidates the selects to:
+ * (select) <-- (assign)*
+ *
+ */
+public class ConsolidateSelectsRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+    	AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+        SelectOperator firstSelect = (SelectOperator) op;
+
+        IFunctionInfo andFn = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND);
+        // New conjuncts for consolidated select.
+        AbstractFunctionCallExpression conj = null;        
+        AbstractLogicalOperator topMostOp = null;
+        AbstractLogicalOperator selectParent = null;
+        AbstractLogicalOperator nextSelect = firstSelect;
+		do {
+        	// Skip through assigns.
+            do {
+            	selectParent = nextSelect;
+            	nextSelect = (AbstractLogicalOperator) selectParent.getInputs().get(0).getValue();
+            } while (nextSelect.getOperatorTag() == LogicalOperatorTag.ASSIGN);
+            // Stop if the child op is not a select.
+            if (nextSelect.getOperatorTag() != LogicalOperatorTag.SELECT) {
+        		break;
+        	}
+            // Remember the top-most op that we are not removing.
+            topMostOp = selectParent;
+            
+            // Initialize the new conjuncts, if necessary.
+            if (conj == null) {
+            	conj = new ScalarFunctionCallExpression(andFn);
+            	// Add the first select's condition.
+            	conj.getArguments().add(new MutableObject<ILogicalExpression>(firstSelect.getCondition().getValue()));
+            }
+            
+            // Consolidate all following selects.
+            do {
+                // Add the condition nextSelect to the new list of conjuncts.
+                conj.getArguments().add(((SelectOperator) nextSelect).getCondition());
+                selectParent = nextSelect;
+                nextSelect = (AbstractLogicalOperator) nextSelect.getInputs().get(0).getValue();
+            } while (nextSelect.getOperatorTag() == LogicalOperatorTag.SELECT);
+            
+            // Hook up the input of the top-most remaining op if necessary.
+            if (topMostOp.getOperatorTag() == LogicalOperatorTag.ASSIGN || topMostOp == firstSelect) {
+            	topMostOp.getInputs().set(0, selectParent.getInputs().get(0));
+            }
+            
+            // Prepare for next iteration.
+            nextSelect = selectParent;
+        } while (true);
+		
+		// Did we consolidate any selects?
+        if (conj == null) {
+        	return false;
+        }
+        
+        // Set the new conjuncts.
+        firstSelect.getCondition().setValue(conj);
+        context.computeAndSetTypeEnvironmentForOperator(firstSelect);
+        return true;
+    }
+}

[36/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
deleted file mode 100644
index adb7a8f..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.OperatorDeepCopyVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-
-public class OperatorManipulationUtil {
-
-    // Transforms all NestedTupleSource operators to EmptyTupleSource operators
-    public static void ntsToEts(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
-            context.computeAndSetTypeEnvironmentForOperator(ets);
-            opRef.setValue(ets);
-        } else {
-            for (Mutable<ILogicalOperator> i : opRef.getValue().getInputs()) {
-                ntsToEts(i, context);
-            }
-        }
-    }
-
-    public static ILogicalOperator eliminateSingleSubplanOverEts(SubplanOperator subplan) {
-        if (subplan.getNestedPlans().size() > 1) {
-            // not a single subplan
-            List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
-            subInpList.clear();
-            subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
-            return subplan;
-        }
-        ILogicalPlan plan = subplan.getNestedPlans().get(0);
-        if (plan.getRoots().size() > 1) {
-            // not a single subplan
-            List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
-            subInpList.clear();
-            subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
-            return subplan;
-        }
-        return plan.getRoots().get(0).getValue();
-    }
-
-    public static boolean setOperatorMode(AbstractLogicalOperator op) {
-        boolean change = false;
-        switch (op.getOperatorTag()) {
-            case DATASOURCESCAN: {
-                op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
-                AbstractLogicalOperator currentOp = op;
-                while (currentOp.getInputs().size() == 1) {
-                    AbstractLogicalOperator child = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue();
-                    if (child.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-                        break;
-                    }
-                    child.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
-                    currentOp = child;
-                }
-                change = true;
-                break;
-            }
-            case NESTEDTUPLESOURCE: {
-                NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
-                AbstractLogicalOperator prevOp = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
-                        .getInputs().get(0).getValue();
-                if (prevOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
-                    nts.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
-                    change = true;
-                }
-                break;
-            }
-            default: {
-                boolean forceUnpartitioned = false;
-                if (op.getOperatorTag() == LogicalOperatorTag.LIMIT) {
-                    LimitOperator opLim = (LimitOperator) op;
-                    if (opLim.isTopmostLimitOp()) {
-                        opLim.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
-                        change = true;
-                        forceUnpartitioned = true;
-                    }
-                }
-                if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
-                    AggregateOperator aggOp = (AggregateOperator) op;
-                    if (aggOp.isGlobal()) {
-                        op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
-                        change = true;
-                        forceUnpartitioned = true;
-                    }
-                }
-
-                for (Mutable<ILogicalOperator> i : op.getInputs()) {
-                    boolean exit = false;
-                    AbstractLogicalOperator inputOp = (AbstractLogicalOperator) i.getValue();
-                    switch (inputOp.getExecutionMode()) {
-                        case PARTITIONED: {
-                            if (forceUnpartitioned)
-                                break;
-                            op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
-                            change = true;
-                            exit = true;
-                            break;
-                        }
-                        case LOCAL: {
-                            op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
-                            change = true;
-                            break;
-                        }
-                    }
-                    if (exit) {
-                        break;
-                    }
-                }
-                break;
-            }
-        }
-        return change;
-    }
-
-    public static void substituteVarRec(AbstractLogicalOperator op, LogicalVariable v1, LogicalVariable v2,
-            boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
-        VariableUtilities.substituteVariables(op, v1, v2, goThroughNts, ctx);
-        for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
-            substituteVarRec((AbstractLogicalOperator) opRef2.getValue(), v1, v2, goThroughNts, ctx);
-        }
-        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE && goThroughNts) {
-            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
-            if (nts.getDataSourceReference() != null) {
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
-                        .getInputs().get(0).getValue();
-                substituteVarRec(op2, v1, v2, goThroughNts, ctx);
-            }
-        }
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans aonp = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : aonp.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> ref : p.getRoots()) {
-                    AbstractLogicalOperator aop = (AbstractLogicalOperator) ref.getValue();
-                    substituteVarRec(aop, v1, v2, goThroughNts, ctx);
-                }
-            }
-        }
-    }
-
-    public static ILogicalPlan deepCopy(ILogicalPlan plan) throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> roots = plan.getRoots();
-        List<Mutable<ILogicalOperator>> newRoots = clonePipeline(roots);
-        return new ALogicalPlanImpl(newRoots);
-    }
-
-    public static ILogicalPlan deepCopy(ILogicalPlan plan, IOptimizationContext ctx) throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> roots = plan.getRoots();
-        List<Mutable<ILogicalOperator>> newRoots = clonePipeline(roots);
-        cloneTypeEnvironments(ctx, roots, newRoots);
-        return new ALogicalPlanImpl(newRoots);
-    }
-
-    private static List<Mutable<ILogicalOperator>> clonePipeline(List<Mutable<ILogicalOperator>> roots)
-            throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> newRoots = new ArrayList<Mutable<ILogicalOperator>>();
-        for (Mutable<ILogicalOperator> opRef : roots) {
-            newRoots.add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(opRef.getValue())));
-        }
-        return newRoots;
-    }
-
-    private static void cloneTypeEnvironments(IOptimizationContext ctx, List<Mutable<ILogicalOperator>> roots,
-            List<Mutable<ILogicalOperator>> newRoots) {
-        for (int i = 0; i < newRoots.size(); i++) {
-            Mutable<ILogicalOperator> opRef = newRoots.get(i);
-            Mutable<ILogicalOperator> oldOpRef = roots.get(i);
-            while (opRef.getValue().getInputs().size() > 0) {
-                ctx.setOutputTypeEnvironment(opRef.getValue(), ctx.getOutputTypeEnvironment(oldOpRef.getValue()));
-                opRef = opRef.getValue().getInputs().get(0);
-                oldOpRef = oldOpRef.getValue().getInputs().get(0);
-            }
-            ctx.setOutputTypeEnvironment(opRef.getValue(), ctx.getOutputTypeEnvironment(oldOpRef.getValue()));
-        }
-    }
-
-    public static ILogicalOperator bottomUpCopyOperators(ILogicalOperator op) throws AlgebricksException {
-        ILogicalOperator newOp = deepCopy(op);
-        newOp.getInputs().clear();
-        for (Mutable<ILogicalOperator> child : op.getInputs())
-            newOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(child.getValue())));
-        return newOp;
-    }
-
-    public static ILogicalOperator deepCopy(ILogicalOperator op) throws AlgebricksException {
-        OperatorDeepCopyVisitor visitor = new OperatorDeepCopyVisitor();
-        return op.accept(visitor, null);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
deleted file mode 100644
index ec9b403..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.util;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-public class OperatorPropertiesUtil {
-
-    public static <T> boolean disjoint(Collection<T> c1, Collection<T> c2) {
-        for (T m : c1) {
-            if (c2.contains(m)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    // Obs: doesn't return expected result for op. with nested plans.
-    private static void getFreeVariablesInOp(ILogicalOperator op, Set<LogicalVariable> freeVars)
-            throws AlgebricksException {
-        VariableUtilities.getUsedVariables(op, freeVars);
-        HashSet<LogicalVariable> produced = new HashSet<LogicalVariable>();
-        VariableUtilities.getProducedVariables(op, produced);
-        for (LogicalVariable v : produced) {
-            freeVars.remove(v);
-        }
-    }
-
-    /**
-     * Adds the free variables of the plan rooted at that operator to the
-     * collection provided.
-     * 
-     * @param op
-     * @param vars
-     *            - The collection to which the free variables will be added.
-     */
-    public static void getFreeVariablesInSelfOrDesc(AbstractLogicalOperator op, Set<LogicalVariable> freeVars)
-            throws AlgebricksException {
-        HashSet<LogicalVariable> produced = new HashSet<LogicalVariable>();
-        VariableUtilities.getProducedVariables(op, produced);
-        for (LogicalVariable v : produced) {
-            freeVars.remove(v);
-        }
-
-        HashSet<LogicalVariable> used = new HashSet<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op, used);
-        for (LogicalVariable v : used) {
-            if (!freeVars.contains(v)) {
-                freeVars.add(v);
-            }
-        }
-
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans s = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : s.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
-                }
-            }
-            s.getUsedVariablesExceptNestedPlans(freeVars);
-            HashSet<LogicalVariable> produced2 = new HashSet<LogicalVariable>();
-            s.getProducedVariablesExceptNestedPlans(produced2);
-            freeVars.removeAll(produced);
-        }
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) i.getValue(), freeVars);
-        }
-    }
-
-    /**
-     * Adds the free variables of the operator path from
-     * op to dest, where dest is a direct/indirect input operator of op in the query plan.
-     * 
-     * @param op
-     *            , the start operator.
-     * @param dest
-     *            , the destination operator (a direct/indirect input operator).
-     * @param freeVars
-     *            - The collection to which the free variables will be added.
-     */
-    public static void getFreeVariablesInPath(ILogicalOperator op, ILogicalOperator dest, Set<LogicalVariable> freeVars)
-            throws AlgebricksException {
-        Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
-        VariableUtilities.getLiveVariables(op, freeVars);
-        collectUsedAndProducedVariablesInPath(op, dest, freeVars, producedVars);
-        freeVars.removeAll(producedVars);
-    }
-
-    /**
-     * @param op
-     *            , the start operator.
-     * @param dest
-     *            , the destination operator (a direct/indirect input operator).
-     * @param usedVars
-     *            , the collection of used variables.
-     * @param producedVars
-     *            , the collection of produced variables.
-     * @return if the current operator is on the path from the original start operator to the destination operator.
-     * @throws AlgebricksException
-     */
-    private static boolean collectUsedAndProducedVariablesInPath(ILogicalOperator op, ILogicalOperator dest,
-            Set<LogicalVariable> usedVars, Set<LogicalVariable> producedVars) throws AlgebricksException {
-        if (op == dest) {
-            return true;
-        }
-        boolean onPath = false;
-        if (((AbstractLogicalOperator) op).hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : a.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    if (isDestInNestedPath((AbstractLogicalOperator) r.getValue(), dest)) {
-                        onPath = true;
-                    }
-                }
-            }
-        }
-        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
-            if (collectUsedAndProducedVariablesInPath(childRef.getValue(), dest, usedVars, producedVars)) {
-                onPath = true;
-            }
-        }
-        if (onPath) {
-            VariableUtilities.getUsedVariables(op, usedVars);
-            VariableUtilities.getProducedVariables(op, producedVars);
-        }
-        return onPath;
-    }
-
-    /***
-     * Recursively checks if the dest operator is in the path of a nested plan
-     * 
-     * @param op
-     * @param dest
-     * @return
-     */
-    private static boolean isDestInNestedPath(AbstractLogicalOperator op, ILogicalOperator dest) {
-        if (op == dest) {
-            return true;
-        }
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            if (isDestInNestedPath((AbstractLogicalOperator) i.getValue(), dest)) {
-                return true;
-            }
-        }
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : a.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    if (isDestInNestedPath((AbstractLogicalOperator) r.getValue(), dest)) {
-                        return true;
-                    }
-                }
-            }
-        }
-        return false;
-    }
-
-    public static void getFreeVariablesInSubplans(AbstractOperatorWithNestedPlans op, Set<LogicalVariable> freeVars)
-            throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
-            }
-        }
-    }
-
-    public static boolean hasFreeVariablesInSelfOrDesc(AbstractLogicalOperator op) throws AlgebricksException {
-        HashSet<LogicalVariable> free = new HashSet<LogicalVariable>();
-        getFreeVariablesInSelfOrDesc(op, free);
-        return !free.isEmpty();
-    }
-
-    public static boolean hasFreeVariables(ILogicalOperator op) throws AlgebricksException {
-        HashSet<LogicalVariable> free = new HashSet<LogicalVariable>();
-        getFreeVariablesInOp(op, free);
-        return !free.isEmpty();
-    }
-
-    public static void computeSchemaAndPropertiesRecIfNull(AbstractLogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        if (op.getSchema() == null) {
-            for (Mutable<ILogicalOperator> i : op.getInputs()) {
-                computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) i.getValue(), context);
-            }
-            if (op.hasNestedPlans()) {
-                AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
-                for (ILogicalPlan p : a.getNestedPlans()) {
-                    for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                        computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) r.getValue(), context);
-                    }
-                }
-            }
-            op.recomputeSchema();
-            op.computeDeliveredPhysicalProperties(context);
-        }
-    }
-
-    public static void computeSchemaRecIfNull(AbstractLogicalOperator op) throws AlgebricksException {
-        if (op.getSchema() == null) {
-            for (Mutable<ILogicalOperator> i : op.getInputs()) {
-                computeSchemaRecIfNull((AbstractLogicalOperator) i.getValue());
-            }
-            if (op.hasNestedPlans()) {
-                AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
-                for (ILogicalPlan p : a.getNestedPlans()) {
-                    for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                        computeSchemaRecIfNull((AbstractLogicalOperator) r.getValue());
-                    }
-                }
-            }
-            op.recomputeSchema();
-        }
-    }
-
-    public static boolean isNullTest(AbstractLogicalOperator op) {
-        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
-            return false;
-        }
-        AbstractLogicalOperator doubleUnder = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        if (doubleUnder.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            return false;
-        }
-        ILogicalExpression eu = ((SelectOperator) op).getCondition().getValue();
-        if (eu.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) eu;
-        if (!f1.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
-            return false;
-        }
-        ILogicalExpression a1 = f1.getArguments().get(0).getValue();
-        if (!a1.getExpressionTag().equals(LogicalExpressionTag.FUNCTION_CALL)) {
-            return false;
-        }
-        AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
-        if (!f2.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
-            return false;
-        }
-        return true;
-    }
-
-    public static void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> r : p.getRoots()) {
-            typeOpRec(r, context);
-        }
-    }
-
-    public static void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            typeOpRec(i, context);
-        }
-        if (op.hasNestedPlans()) {
-            for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
-                typePlan(p, context);
-            }
-        }
-        context.computeAndSetTypeEnvironmentForOperator(op);
-    }
-
-    public static boolean isAlwaysTrueCond(ILogicalExpression cond) {
-        if (cond.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
-            return ((ConstantExpression) cond).getValue().isTrue();
-        }
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
deleted file mode 100644
index c7424f0..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-
-public abstract class AbstractConstVarFunVisitor<R, T> implements ILogicalExpressionVisitor<R, T> {
-
-    public abstract R visitFunctionCallExpression(AbstractFunctionCallExpression expr, T arg)
-            throws AlgebricksException;
-
-    public R visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, T arg)
-            throws AlgebricksException {
-        return visitFunctionCallExpression(expr, arg);
-    }
-
-    public R visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, T arg) throws AlgebricksException {
-        return visitFunctionCallExpression(expr, arg);
-    }
-
-    public R visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, T arg) throws AlgebricksException {
-        return visitFunctionCallExpression(expr, arg);
-    }
-
-    public R visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, T arg)
-            throws AlgebricksException {
-        return visitFunctionCallExpression(expr, arg);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
deleted file mode 100644
index 06cec65..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-
-public interface ILogicalExpressionReferenceTransform {
-    // returns true if any change is made to the expression
-    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
deleted file mode 100644
index 3448db8..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-
-public interface ILogicalExpressionVisitor<R, T> {
-    public R visitConstantExpression(ConstantExpression expr, T arg) throws AlgebricksException;
-
-    public R visitVariableReferenceExpression(VariableReferenceExpression expr, T arg) throws AlgebricksException;
-
-    public R visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, T arg)
-            throws AlgebricksException;
-
-    public R visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, T arg) throws AlgebricksException;
-
-    public R visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, T arg) throws AlgebricksException;
-
-    public R visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, T arg)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
deleted file mode 100644
index 359e243..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-
-public interface ILogicalOperatorVisitor<R, T> {
-
-    public R visitAggregateOperator(AggregateOperator op, T arg) throws AlgebricksException;
-
-    public R visitRunningAggregateOperator(RunningAggregateOperator op, T arg) throws AlgebricksException;
-
-    public R visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, T arg) throws AlgebricksException;
-
-    public R visitGroupByOperator(GroupByOperator op, T arg) throws AlgebricksException;
-
-    public R visitLimitOperator(LimitOperator op, T arg) throws AlgebricksException;
-
-    public R visitInnerJoinOperator(InnerJoinOperator op, T arg) throws AlgebricksException;
-
-    public R visitLeftOuterJoinOperator(LeftOuterJoinOperator op, T arg) throws AlgebricksException;
-
-    public R visitNestedTupleSourceOperator(NestedTupleSourceOperator op, T arg) throws AlgebricksException;
-
-    public R visitOrderOperator(OrderOperator op, T arg) throws AlgebricksException;
-
-    public R visitAssignOperator(AssignOperator op, T arg) throws AlgebricksException;
-
-    public R visitSelectOperator(SelectOperator op, T arg) throws AlgebricksException;
-
-    public R visitExtensionOperator(ExtensionOperator op, T arg) throws AlgebricksException;
-
-    public R visitProjectOperator(ProjectOperator op, T arg) throws AlgebricksException;
-
-    public R visitPartitioningSplitOperator(PartitioningSplitOperator op, T arg) throws AlgebricksException;
-
-    public R visitReplicateOperator(ReplicateOperator op, T arg) throws AlgebricksException;
-
-    public R visitMaterializeOperator(MaterializeOperator op, T arg) throws AlgebricksException;
-
-    public R visitScriptOperator(ScriptOperator op, T arg) throws AlgebricksException;
-
-    public R visitSubplanOperator(SubplanOperator op, T arg) throws AlgebricksException;
-
-    public R visitSinkOperator(SinkOperator op, T arg) throws AlgebricksException;
-
-    public R visitUnionOperator(UnionAllOperator op, T arg) throws AlgebricksException;
-
-    public R visitUnnestOperator(UnnestOperator op, T arg) throws AlgebricksException;
-
-    public R visitUnnestMapOperator(UnnestMapOperator op, T arg) throws AlgebricksException;
-
-    public R visitDataScanOperator(DataSourceScanOperator op, T arg) throws AlgebricksException;
-
-    public R visitDistinctOperator(DistinctOperator op, T arg) throws AlgebricksException;
-
-    public R visitExchangeOperator(ExchangeOperator op, T arg) throws AlgebricksException;
-
-    public R visitWriteOperator(WriteOperator op, T arg) throws AlgebricksException;
-
-    public R visitDistributeResultOperator(DistributeResultOperator op, T arg) throws AlgebricksException;
-
-    public R visitWriteResultOperator(WriteResultOperator op, T arg) throws AlgebricksException;
-
-    public R visitInsertDeleteOperator(InsertDeleteOperator op, T tag) throws AlgebricksException;
-
-    public R visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, T tag) throws AlgebricksException;
-
-    public R visitExternalDataLookupOperator(ExternalDataLookupOperator op, T arg) throws AlgebricksException;
-
-    public R visitTokenizeOperator(TokenizeOperator op, T arg) throws AlgebricksException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
deleted file mode 100644
index abdd6d1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.config;
-
-import java.util.logging.Logger;
-
-public class AlgebricksConfig {
-    public static final boolean DEBUG = true;
-    public static final String ALGEBRICKS_LOGGER_NAME = "edu.uci.ics.hyracks.algebricks";
-    public static final Logger ALGEBRICKS_LOGGER = Logger.getLogger(ALGEBRICKS_LOGGER_NAME);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java
deleted file mode 100644
index d449f92..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.config;
-
-import java.util.logging.Formatter;
-import java.util.logging.Handler;
-import java.util.logging.LogRecord;
-
-public class SysoutFormatter extends Formatter {
-    public String format(LogRecord rec) {
-        StringBuffer buf = new StringBuffer(5000);
-        buf.append(formatMessage(rec));
-        return buf.toString();
-    }
-
-    public String getHead(Handler h) {
-        return "";
-    }
-
-    public String getTail(Handler h) {
-        return "";
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
deleted file mode 100644
index 4cb111d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/ConnectorPolicyAssignmentPolicy.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
-
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
-import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
-import edu.uci.ics.hyracks.api.dataflow.connectors.PipeliningConnectorPolicy;
-import edu.uci.ics.hyracks.api.dataflow.connectors.SendSideMaterializedPipeliningConnectorPolicy;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
-
-public class ConnectorPolicyAssignmentPolicy implements IConnectorPolicyAssignmentPolicy {
-    private static final long serialVersionUID = 1L;
-    private IConnectorPolicy senderSideMaterializePolicy = new SendSideMaterializedPipeliningConnectorPolicy();
-    private IConnectorPolicy pipeliningPolicy = new PipeliningConnectorPolicy();
-
-    @Override
-    public IConnectorPolicy getConnectorPolicyAssignment(IConnectorDescriptor c, int nProducers, int nConsumers,
-            int[] fanouts) {
-        if (c instanceof MToNPartitioningMergingConnectorDescriptor) {
-            return senderSideMaterializePolicy;
-        } else {
-            return pipeliningPolicy;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
deleted file mode 100644
index e1f897c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class JobBuilder implements IHyracksJobBuilder {
-
-    private JobSpecification jobSpec;
-    private AlgebricksPartitionConstraint clusterLocations;
-
-    private Map<ILogicalOperator, ArrayList<ILogicalOperator>> outEdges = new HashMap<ILogicalOperator, ArrayList<ILogicalOperator>>();
-    private Map<ILogicalOperator, ArrayList<ILogicalOperator>> inEdges = new HashMap<ILogicalOperator, ArrayList<ILogicalOperator>>();
-    private Map<ILogicalOperator, Pair<IConnectorDescriptor, TargetConstraint>> connectors = new HashMap<ILogicalOperator, Pair<IConnectorDescriptor, TargetConstraint>>();
-
-    private Map<ILogicalOperator, Pair<IPushRuntimeFactory, RecordDescriptor>> microOps = new HashMap<ILogicalOperator, Pair<IPushRuntimeFactory, RecordDescriptor>>();
-    private Map<IPushRuntimeFactory, ILogicalOperator> revMicroOpMap = new HashMap<IPushRuntimeFactory, ILogicalOperator>();
-    private Map<ILogicalOperator, IOperatorDescriptor> hyracksOps = new HashMap<ILogicalOperator, IOperatorDescriptor>();
-    private Map<ILogicalOperator, AlgebricksPartitionConstraint> pcForMicroOps = new HashMap<ILogicalOperator, AlgebricksPartitionConstraint>();
-
-    private int aodCounter = 0;
-    private Map<ILogicalOperator, Integer> algebraicOpBelongingToMetaAsterixOp = new HashMap<ILogicalOperator, Integer>();
-    private Map<Integer, List<Pair<IPushRuntimeFactory, RecordDescriptor>>> metaAsterixOpSkeletons = new HashMap<Integer, List<Pair<IPushRuntimeFactory, RecordDescriptor>>>();
-    private Map<Integer, AlgebricksMetaOperatorDescriptor> metaAsterixOps = new HashMap<Integer, AlgebricksMetaOperatorDescriptor>();
-    private final Map<IOperatorDescriptor, AlgebricksPartitionConstraint> partitionConstraintMap = new HashMap<IOperatorDescriptor, AlgebricksPartitionConstraint>();
-
-    public JobBuilder(JobSpecification jobSpec, AlgebricksPartitionConstraint clusterLocations) {
-        this.jobSpec = jobSpec;
-        this.clusterLocations = clusterLocations;
-    }
-
-    @Override
-    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc) {
-        contributeMicroOperator(op, runtime, recDesc, null);
-    }
-
-    @Override
-    public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc,
-            AlgebricksPartitionConstraint pc) {
-        microOps.put(op, new Pair<IPushRuntimeFactory, RecordDescriptor>(runtime, recDesc));
-        revMicroOpMap.put(runtime, op);
-        if (pc != null) {
-            pcForMicroOps.put(op, pc);
-        }
-        AbstractLogicalOperator logicalOp = (AbstractLogicalOperator) op;
-        if (logicalOp.getExecutionMode() == ExecutionMode.UNPARTITIONED && pc == null) {
-            AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1);
-            pcForMicroOps.put(logicalOp, apc);
-        }
-    }
-
-    @Override
-    public void contributeConnector(ILogicalOperator exchgOp, IConnectorDescriptor conn) {
-        connectors.put(exchgOp, new Pair<IConnectorDescriptor, TargetConstraint>(conn, null));
-    }
-
-    @Override
-    public void contributeConnectorWithTargetConstraint(ILogicalOperator exchgOp, IConnectorDescriptor conn,
-            TargetConstraint numberOfTargetPartitions) {
-        connectors.put(exchgOp, new Pair<IConnectorDescriptor, TargetConstraint>(conn, numberOfTargetPartitions));
-    }
-
-    @Override
-    public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex) {
-        ArrayList<ILogicalOperator> outputs = outEdges.get(src);
-        if (outputs == null) {
-            outputs = new ArrayList<ILogicalOperator>();
-            outEdges.put(src, outputs);
-        }
-        addAtPos(outputs, dest, srcOutputIndex);
-
-        ArrayList<ILogicalOperator> inp = inEdges.get(dest);
-        if (inp == null) {
-            inp = new ArrayList<ILogicalOperator>();
-            inEdges.put(dest, inp);
-        }
-        addAtPos(inp, src, destInputIndex);
-    }
-
-    @Override
-    public void contributeHyracksOperator(ILogicalOperator op, IOperatorDescriptor opDesc) {
-        hyracksOps.put(op, opDesc);
-    }
-
-    @Override
-    public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apc) {
-        partitionConstraintMap.put(opDesc, apc);
-    }
-
-    @Override
-    public JobSpecification getJobSpec() {
-        return jobSpec;
-    }
-
-    @Override
-    public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException {
-        buildAsterixComponents();
-        Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = setupConnectors();
-        for (ILogicalOperator r : roots) {
-            IOperatorDescriptor opDesc = findOpDescForAlgebraicOp(r);
-            jobSpec.addRoot(opDesc);
-        }
-        setAllPartitionConstraints(tgtConstraints);
-    }
-
-    private void setAllPartitionConstraints(Map<IConnectorDescriptor, TargetConstraint> tgtConstraints) {
-        List<OperatorDescriptorId> roots = jobSpec.getRoots();
-        setSpecifiedPartitionConstraints();
-        for (OperatorDescriptorId rootId : roots) {
-            setPartitionConstraintsDFS(rootId, tgtConstraints, null);
-        }
-    }
-
-    private void setSpecifiedPartitionConstraints() {
-        for (ILogicalOperator op : pcForMicroOps.keySet()) {
-            AlgebricksPartitionConstraint pc = pcForMicroOps.get(op);
-            Integer k = algebraicOpBelongingToMetaAsterixOp.get(op);
-            AlgebricksMetaOperatorDescriptor amod = metaAsterixOps.get(k);
-            partitionConstraintMap.put(amod, pc);
-        }
-        for (IOperatorDescriptor opDesc : partitionConstraintMap.keySet()) {
-            AlgebricksPartitionConstraint pc = partitionConstraintMap.get(opDesc);
-            AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, pc);
-        }
-    }
-
-    private void setPartitionConstraintsDFS(OperatorDescriptorId opId,
-            Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp) {
-        List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
-        AlgebricksPartitionConstraint opConstraint = null;
-        IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
-        if (opInputs != null) {
-            for (IConnectorDescriptor conn : opInputs) {
-                ConnectorDescriptorId cid = conn.getConnectorId();
-                org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec
-                        .getConnectorOperatorMap().get(cid);
-                IOperatorDescriptor src = p.getLeft().getLeft();
-                // DFS
-                setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc);
-
-                TargetConstraint constraint = tgtConstraints.get(conn);
-                if (constraint != null) {
-                    switch (constraint) {
-                        case ONE: {
-                            opConstraint = new AlgebricksCountPartitionConstraint(1);
-                            break;
-                        }
-                        case SAME_COUNT: {
-                            opConstraint = partitionConstraintMap.get(src);
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-        if (partitionConstraintMap.get(opDesc) == null) {
-            if (opConstraint == null) {
-                if (parentOp != null) {
-                    AlgebricksPartitionConstraint pc = partitionConstraintMap.get(parentOp);
-                    if (pc != null) {
-                        opConstraint = pc;
-                    } else if (opInputs == null || opInputs.size() == 0) {
-                        opConstraint = new AlgebricksCountPartitionConstraint(1);
-                    }
-                }
-                if (opConstraint == null) {
-                    opConstraint = clusterLocations;
-                }
-            }
-            partitionConstraintMap.put(opDesc, opConstraint);
-            AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, opConstraint);
-        }
-    }
-
-    private Map<IConnectorDescriptor, TargetConstraint> setupConnectors() throws AlgebricksException {
-        Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = new HashMap<IConnectorDescriptor, TargetConstraint>();
-        for (ILogicalOperator exchg : connectors.keySet()) {
-            ILogicalOperator inOp = inEdges.get(exchg).get(0);
-            ILogicalOperator outOp = outEdges.get(exchg).get(0);
-            IOperatorDescriptor inOpDesc = findOpDescForAlgebraicOp(inOp);
-            IOperatorDescriptor outOpDesc = findOpDescForAlgebraicOp(outOp);
-            Pair<IConnectorDescriptor, TargetConstraint> connPair = connectors.get(exchg);
-            IConnectorDescriptor conn = connPair.first;
-            int producerPort = outEdges.get(inOp).indexOf(exchg);
-            int consumerPort = inEdges.get(outOp).indexOf(exchg);
-            jobSpec.connect(conn, inOpDesc, producerPort, outOpDesc, consumerPort);
-            if (connPair.second != null) {
-                tgtConstraints.put(conn, connPair.second);
-            }
-        }
-        return tgtConstraints;
-    }
-
-    private IOperatorDescriptor findOpDescForAlgebraicOp(ILogicalOperator op) throws AlgebricksException {
-        IOperatorDescriptor hOpDesc = hyracksOps.get(op);
-        if (hOpDesc != null) {
-            return hOpDesc;
-        }
-        Integer metaOpKey = algebraicOpBelongingToMetaAsterixOp.get(op);
-        if (metaOpKey == null) {
-            throw new AlgebricksException("Could not generate operator descriptor for operator " + op);
-        }
-        return metaAsterixOps.get(metaOpKey);
-    }
-
-    private void buildAsterixComponents() {
-        for (ILogicalOperator aop : microOps.keySet()) {
-            addMicroOpToMetaRuntimeOp(aop);
-        }
-        for (Integer k : metaAsterixOpSkeletons.keySet()) {
-            List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents = metaAsterixOpSkeletons.get(k);
-            AlgebricksMetaOperatorDescriptor amod = buildMetaAsterixOpDesc(opContents);
-            metaAsterixOps.put(k, amod);
-        }
-    }
-
-    private AlgebricksMetaOperatorDescriptor buildMetaAsterixOpDesc(
-            List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents) {
-        // RecordDescriptor outputRecordDesc = null;
-        int n = opContents.size();
-        IPushRuntimeFactory[] runtimeFactories = new IPushRuntimeFactory[n];
-        RecordDescriptor[] internalRecordDescriptors = new RecordDescriptor[n];
-        int i = 0;
-        for (Pair<IPushRuntimeFactory, RecordDescriptor> p : opContents) {
-            runtimeFactories[i] = p.first;
-            internalRecordDescriptors[i] = p.second;
-            // if (i == n - 1) {
-            // outputRecordDesc = p.second;
-            // }
-            i++;
-        }
-        ILogicalOperator lastLogicalOp = revMicroOpMap.get(runtimeFactories[n - 1]);
-        ArrayList<ILogicalOperator> outOps = outEdges.get(lastLogicalOp);
-        int outArity = (outOps == null) ? 0 : outOps.size();
-        ILogicalOperator firstLogicalOp = revMicroOpMap.get(runtimeFactories[0]);
-        ArrayList<ILogicalOperator> inOps = inEdges.get(firstLogicalOp);
-        int inArity = (inOps == null) ? 0 : inOps.size();
-        // boolean isLeafOp = inEdges.get(firstLogicalOp) == null;
-        return new AlgebricksMetaOperatorDescriptor(jobSpec, inArity, outArity, runtimeFactories,
-                internalRecordDescriptors);
-    }
-
-    private void addMicroOpToMetaRuntimeOp(ILogicalOperator aop) {
-        Integer k = algebraicOpBelongingToMetaAsterixOp.get(aop);
-        if (k == null) {
-            k = createNewMetaOpInfo(aop);
-        }
-        ArrayList<ILogicalOperator> destList = outEdges.get(aop);
-        if (destList == null || destList.size() != 1) {
-            // for now, we only support linear plans inside meta-ops.
-            return;
-        }
-        ILogicalOperator dest = destList.get(0);
-        Integer j = algebraicOpBelongingToMetaAsterixOp.get(dest);
-        if (j == null && microOps.get(dest) != null) {
-            algebraicOpBelongingToMetaAsterixOp.put(dest, k);
-            List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
-            aodContent1.add(microOps.get(dest));
-        } else if (j != null && j.intValue() != k.intValue()) {
-            // merge the j component into the k component
-            List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
-            List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent2 = metaAsterixOpSkeletons.get(j);
-            aodContent1.addAll(aodContent2);
-            metaAsterixOpSkeletons.remove(j);
-            for (ILogicalOperator m : algebraicOpBelongingToMetaAsterixOp.keySet()) {
-                Integer g = algebraicOpBelongingToMetaAsterixOp.get(m);
-                if (g.intValue() == j.intValue()) {
-                    algebraicOpBelongingToMetaAsterixOp.put(m, k);
-                }
-            }
-        }
-
-    }
-
-    private int createNewMetaOpInfo(ILogicalOperator aop) {
-        int n = aodCounter;
-        aodCounter++;
-        List<Pair<IPushRuntimeFactory, RecordDescriptor>> metaOpContents = new ArrayList<Pair<IPushRuntimeFactory, RecordDescriptor>>();
-        metaOpContents.add(microOps.get(aop));
-        metaAsterixOpSkeletons.put(n, metaOpContents);
-        algebraicOpBelongingToMetaAsterixOp.put(aop, n);
-        return n;
-    }
-
-    private <E> void addAtPos(ArrayList<E> a, E elem, int pos) {
-        int n = a.size();
-        if (n > pos) {
-            a.set(pos, elem);
-        } else {
-            for (int k = n; k < pos; k++) {
-                a.add(null);
-            }
-            a.add(elem);
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
deleted file mode 100644
index a181304..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenContext.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFamilyProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryIntegerInspectorFactory;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IPredicateEvaluatorFactoryProvider;
-
-public class JobGenContext {
-    private final IOperatorSchema outerFlowSchema;
-    private final Map<ILogicalOperator, IOperatorSchema> schemaMap = new HashMap<ILogicalOperator, IOperatorSchema>();
-    private final ISerializerDeserializerProvider serializerDeserializerProvider;
-    private final IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
-    private final IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider;
-    private final IBinaryComparatorFactoryProvider comparatorFactoryProvider;
-    private final IPrinterFactoryProvider printerFactoryProvider;
-    private final ITypeTraitProvider typeTraitProvider;
-    private final IMetadataProvider<?, ?> metadataProvider;
-    private final INullWriterFactory nullWriterFactory;
-    private final INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
-    private final Object appContext;
-    private final IBinaryBooleanInspectorFactory booleanInspectorFactory;
-    private final IBinaryIntegerInspectorFactory integerInspectorFactory;
-    private final IExpressionRuntimeProvider expressionRuntimeProvider;
-    private final IExpressionTypeComputer expressionTypeComputer;
-    private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
-    private final IPartialAggregationTypeComputer partialAggregationTypeComputer;
-    private final IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider;
-    private final int frameSize;
-    private AlgebricksPartitionConstraint clusterLocations;
-    private int varCounter;
-    private final ITypingContext typingContext;
-
-    public JobGenContext(IOperatorSchema outerFlowSchema, IMetadataProvider<?, ?> metadataProvider, Object appContext,
-            ISerializerDeserializerProvider serializerDeserializerProvider,
-            IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider,
-            IBinaryHashFunctionFamilyProvider hashFunctionFamilyProvider,
-            IBinaryComparatorFactoryProvider comparatorFactoryProvider, ITypeTraitProvider typeTraitProvider,
-            IBinaryBooleanInspectorFactory booleanInspectorFactory,
-            IBinaryIntegerInspectorFactory integerInspectorFactory, IPrinterFactoryProvider printerFactoryProvider,
-            INullWriterFactory nullWriterFactory,
-            INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider,
-            IExpressionRuntimeProvider expressionRuntimeProvider, IExpressionTypeComputer expressionTypeComputer,
-            INullableTypeComputer nullableTypeComputer, ITypingContext typingContext,
-            IExpressionEvalSizeComputer expressionEvalSizeComputer,
-            IPartialAggregationTypeComputer partialAggregationTypeComputer,
-            IPredicateEvaluatorFactoryProvider predEvaluatorFactoryProvider, int frameSize,
-            AlgebricksPartitionConstraint clusterLocations) {
-        this.outerFlowSchema = outerFlowSchema;
-        this.metadataProvider = metadataProvider;
-        this.appContext = appContext;
-        this.serializerDeserializerProvider = serializerDeserializerProvider;
-        this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
-        this.hashFunctionFamilyProvider = hashFunctionFamilyProvider;
-        this.comparatorFactoryProvider = comparatorFactoryProvider;
-        this.typeTraitProvider = typeTraitProvider;
-        this.booleanInspectorFactory = booleanInspectorFactory;
-        this.integerInspectorFactory = integerInspectorFactory;
-        this.printerFactoryProvider = printerFactoryProvider;
-        this.clusterLocations = clusterLocations;
-        this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
-        this.nullWriterFactory = nullWriterFactory;
-        this.expressionRuntimeProvider = expressionRuntimeProvider;
-        this.expressionTypeComputer = expressionTypeComputer;
-        this.typingContext = typingContext;
-        this.expressionEvalSizeComputer = expressionEvalSizeComputer;
-        this.partialAggregationTypeComputer = partialAggregationTypeComputer;
-        this.predEvaluatorFactoryProvider = predEvaluatorFactoryProvider;
-        this.frameSize = frameSize;
-        this.varCounter = 0;
-    }
-
-    public IOperatorSchema getOuterFlowSchema() {
-        return outerFlowSchema;
-    }
-
-    public AlgebricksPartitionConstraint getClusterLocations() {
-        return clusterLocations;
-    }
-
-    public IMetadataProvider<?, ?> getMetadataProvider() {
-        return metadataProvider;
-    }
-
-    public Object getAppContext() {
-        return appContext;
-    }
-
-    public ISerializerDeserializerProvider getSerializerDeserializerProvider() {
-        return serializerDeserializerProvider;
-    }
-
-    public IBinaryHashFunctionFactoryProvider getBinaryHashFunctionFactoryProvider() {
-        return hashFunctionFactoryProvider;
-    }
-
-    public IBinaryHashFunctionFamilyProvider getBinaryHashFunctionFamilyProvider() {
-        return hashFunctionFamilyProvider;
-    }
-
-    public IBinaryComparatorFactoryProvider getBinaryComparatorFactoryProvider() {
-        return comparatorFactoryProvider;
-    }
-
-    public ITypeTraitProvider getTypeTraitProvider() {
-        return typeTraitProvider;
-    }
-
-    public IBinaryBooleanInspectorFactory getBinaryBooleanInspectorFactory() {
-        return booleanInspectorFactory;
-    }
-
-    public IBinaryIntegerInspectorFactory getBinaryIntegerInspectorFactory() {
-        return integerInspectorFactory;
-    }
-
-    public IPrinterFactoryProvider getPrinterFactoryProvider() {
-        return printerFactoryProvider;
-    }
-
-    public IPredicateEvaluatorFactoryProvider getPredicateEvaluatorFactoryProvider() {
-        return predEvaluatorFactoryProvider;
-    }
-
-    public IExpressionRuntimeProvider getExpressionRuntimeProvider() {
-        return expressionRuntimeProvider;
-    }
-
-    public IOperatorSchema getSchema(ILogicalOperator op) {
-        return schemaMap.get(op);
-    }
-
-    public void putSchema(ILogicalOperator op, IOperatorSchema schema) {
-        schemaMap.put(op, schema);
-    }
-
-    public LogicalVariable createNewVar() {
-        varCounter++;
-        LogicalVariable var = new LogicalVariable(-varCounter);
-        return var;
-    }
-
-    public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env) throws AlgebricksException {
-        return expressionTypeComputer.getType(expr, typingContext.getMetadataProvider(), env);
-    }
-
-    public INullWriterFactory getNullWriterFactory() {
-        return nullWriterFactory;
-    }
-
-    public INormalizedKeyComputerFactoryProvider getNormalizedKeyComputerFactoryProvider() {
-        return normalizedKeyComputerFactoryProvider;
-    }
-
-    public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
-        return expressionEvalSizeComputer;
-    }
-
-    public int getFrameSize() {
-        return frameSize;
-    }
-
-    public IPartialAggregationTypeComputer getPartialAggregationTypeComputer() {
-        return partialAggregationTypeComputer;
-    }
-
-    public IVariableTypeEnvironment getTypeEnvironment(ILogicalOperator op) {
-        return typingContext.getOutputTypeEnvironment(op);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
deleted file mode 100644
index 2d195e4..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/JobGenHelper.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFamilyProvider;
-import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public final class JobGenHelper {
-
-    private static final Logger LOGGER = Logger.getLogger(JobGenHelper.class.getName());
-
-    @SuppressWarnings("rawtypes")
-    public static RecordDescriptor mkRecordDescriptor(IVariableTypeEnvironment env, IOperatorSchema opSchema,
-            JobGenContext context) throws AlgebricksException {
-        ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()];
-        ITypeTraits[] typeTraits = new ITypeTraits[opSchema.getSize()];
-        ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider();
-        ITypeTraitProvider ttp = context.getTypeTraitProvider();
-        int i = 0;
-        for (LogicalVariable var : opSchema) {
-            Object t = env.getVarType(var);
-            if (t == null) {
-                LOGGER.warning("No type for variable " + var);
-            }
-            fields[i] = sdp.getSerializerDeserializer(t);
-            typeTraits[i] = ttp.getTypeTrait(t);
-            i++;
-        }
-        return new RecordDescriptor(fields, typeTraits);
-    }
-
-    public static IPrinterFactory[] mkPrinterFactories(IOperatorSchema opSchema, IVariableTypeEnvironment env,
-            JobGenContext context, int[] printColumns) throws AlgebricksException {
-        IPrinterFactory[] pf = new IPrinterFactory[printColumns.length];
-        IPrinterFactoryProvider pff = context.getPrinterFactoryProvider();
-        for (int i = 0; i < pf.length; i++) {
-            LogicalVariable v = opSchema.getVariable(printColumns[i]);
-            Object t = env.getVarType(v);
-            pf[i] = pff.getPrinterFactory(t);
-        }
-        return pf;
-    }
-
-    public static int[] variablesToFieldIndexes(Collection<LogicalVariable> varLogical, IOperatorSchema opSchema) {
-        int[] tuplePos = new int[varLogical.size()];
-        int i = 0;
-        for (LogicalVariable var : varLogical) {
-            tuplePos[i] = opSchema.findVariable(var);
-            i++;
-        }
-        return tuplePos;
-    }
-
-    public static IBinaryHashFunctionFactory[] variablesToBinaryHashFunctionFactories(
-            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
-            throws AlgebricksException {
-        IBinaryHashFunctionFactory[] funFactories = new IBinaryHashFunctionFactory[varLogical.size()];
-        int i = 0;
-        IBinaryHashFunctionFactoryProvider bhffProvider = context.getBinaryHashFunctionFactoryProvider();
-        for (LogicalVariable var : varLogical) {
-            Object type = env.getVarType(var);
-            funFactories[i++] = bhffProvider.getBinaryHashFunctionFactory(type);
-        }
-        return funFactories;
-    }
-
-    public static IBinaryHashFunctionFamily[] variablesToBinaryHashFunctionFamilies(
-            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
-            throws AlgebricksException {
-        IBinaryHashFunctionFamily[] funFamilies = new IBinaryHashFunctionFamily[varLogical.size()];
-        int i = 0;
-        IBinaryHashFunctionFamilyProvider bhffProvider = context.getBinaryHashFunctionFamilyProvider();
-        for (LogicalVariable var : varLogical) {
-            Object type = env.getVarType(var);
-            funFamilies[i++] = bhffProvider.getBinaryHashFunctionFamily(type);
-        }
-        return funFamilies;
-    }
-
-    public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(
-            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
-            throws AlgebricksException {
-        IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[varLogical.size()];
-        IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
-        int i = 0;
-        for (LogicalVariable v : varLogical) {
-            Object type = env.getVarType(v);
-            compFactories[i++] = bcfProvider.getBinaryComparatorFactory(type, true);
-        }
-        return compFactories;
-    }
-
-    public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(List<LogicalVariable> varLogical,
-            int start, int size, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
-        IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[size];
-        IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
-        for (int i = 0; i < size; i++) {
-            Object type = env.getVarType(varLogical.get(start + i));
-            compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true);
-        }
-        return compFactories;
-    }
-
-    public static INormalizedKeyComputerFactory variablesToAscNormalizedKeyComputerFactory(
-            Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
-            throws AlgebricksException {
-        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
-        if (nkcfProvider == null)
-            return null;
-        for (LogicalVariable v : varLogical) {
-            Object type = env.getVarType(v);
-            return nkcfProvider.getNormalizedKeyComputerFactory(type, true);
-        }
-        return null;
-    }
-
-    public static ITypeTraits[] variablesToTypeTraits(Collection<LogicalVariable> varLogical,
-            IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
-        ITypeTraits[] typeTraits = new ITypeTraits[varLogical.size()];
-        ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
-        int i = 0;
-        for (LogicalVariable v : varLogical) {
-            Object type = env.getVarType(v);
-            typeTraits[i++] = typeTraitProvider.getTypeTrait(type);
-        }
-        return typeTraits;
-    }
-
-    public static ITypeTraits[] variablesToTypeTraits(List<LogicalVariable> varLogical, int start, int size,
-            IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
-        ITypeTraits[] typeTraits = new ITypeTraits[size];
-        ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
-        for (int i = 0; i < size; i++) {
-            Object type = env.getVarType(varLogical.get(start + i));
-            typeTraits[i] = typeTraitProvider.getTypeTrait(type);
-        }
-        return typeTraits;
-    }
-
-    public static int[] projectAllVariables(IOperatorSchema opSchema) {
-        int[] projectionList = new int[opSchema.getSize()];
-        int k = 0;
-        for (LogicalVariable v : opSchema) {
-            projectionList[k++] = opSchema.findVariable(v);
-        }
-        return projectionList;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
deleted file mode 100644
index dc1fb35..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/jobgen/impl/OperatorSchemaImpl.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.jobgen.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-
-public class OperatorSchemaImpl implements IOperatorSchema {
-    private final Map<LogicalVariable, Integer> varMap;
-
-    private final List<LogicalVariable> varList;
-
-    public OperatorSchemaImpl() {
-        varMap = new HashMap<LogicalVariable, Integer>();
-        varList = new ArrayList<LogicalVariable>();
-    }
-
-    @Override
-    public void addAllVariables(IOperatorSchema source) {
-        for (LogicalVariable v : source) {
-            varMap.put(v, varList.size());
-            varList.add(v);
-        }
-    }
-
-    @Override
-    public void addAllNewVariables(IOperatorSchema source) {
-        for (LogicalVariable v : source) {
-            if (varMap.get(v) == null) {
-                varMap.put(v, varList.size());
-                varList.add(v);
-            }
-        }
-    }
-
-    @Override
-    public int addVariable(LogicalVariable var) {
-        int idx = varList.size();
-        varMap.put(var, idx);
-        varList.add(var);
-        return idx;
-    }
-
-    @Override
-    public void clear() {
-        varMap.clear();
-        varList.clear();
-    }
-
-    @Override
-    public int findVariable(LogicalVariable var) {
-        Integer i = varMap.get(var);
-        if (i == null) {
-            return -1;
-        }
-        return i;
-    }
-
-    @Override
-    public int getSize() {
-        return varList.size();
-    }
-
-    @Override
-    public LogicalVariable getVariable(int index) {
-        return varList.get(index);
-    }
-
-    @Override
-    public Iterator<LogicalVariable> iterator() {
-        return varList.iterator();
-    }
-
-    @Override
-    public String toString() {
-        return varMap.toString();
-    }
-}
\ No newline at end of file

[18/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
deleted file mode 100644
index 9ae68bb..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime;
-
-import java.io.DataOutput;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.ConstantValue;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions.PigletFunctionRegistry;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.evaluators.ConstantEvalFactory;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-
-public class PigletExpressionJobGen implements ILogicalExpressionJobGen {
-    @Override
-    public ICopyEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
-            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
-        switch (expr.getExpressionTag()) {
-            case CONSTANT: {
-                ConstantValue cv = (ConstantValue) ((ConstantExpression) expr).getValue();
-                Type type = cv.getType();
-                String image = cv.getImage();
-                ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-                DataOutput dos = abvs.getDataOutput();
-                switch (type.getTag()) {
-                    case INTEGER:
-                        try {
-                            IntegerSerializerDeserializer.INSTANCE.serialize(Integer.valueOf(image), dos);
-                        } catch (Exception e) {
-                            throw new AlgebricksException(e);
-                        }
-                        break;
-
-                    case CHAR_ARRAY:
-                        try {
-                            UTF8StringSerializerDeserializer.INSTANCE.serialize(image, dos);
-                        } catch (Exception e) {
-                            throw new AlgebricksException(e);
-                        }
-                        break;
-
-                    default:
-                        throw new UnsupportedOperationException("Unsupported constant type: " + type.getTag());
-                }
-                return new ConstantEvalFactory(Arrays.copyOf(abvs.getByteArray(), abvs.getLength()));
-            }
-
-            case FUNCTION_CALL: {
-                ScalarFunctionCallExpression sfce = (ScalarFunctionCallExpression) expr;
-
-                List<Mutable<ILogicalExpression>> argExprs = sfce.getArguments();
-                ICopyEvaluatorFactory argEvalFactories[] = new ICopyEvaluatorFactory[argExprs.size()];
-                for (int i = 0; i < argEvalFactories.length; ++i) {
-                    Mutable<ILogicalExpression> er = argExprs.get(i);
-                    argEvalFactories[i] = createEvaluatorFactory(er.getValue(), env, inputSchemas, context);
-                }
-                ICopyEvaluatorFactory funcEvalFactory;
-                try {
-                    funcEvalFactory = PigletFunctionRegistry.createFunctionEvaluatorFactory(
-                            sfce.getFunctionIdentifier(), argEvalFactories);
-                } catch (PigletException e) {
-                    throw new AlgebricksException(e);
-                }
-                return funcEvalFactory;
-            }
-
-            case VARIABLE: {
-                LogicalVariable var = ((VariableReferenceExpression) expr).getVariableReference();
-                int index = inputSchemas[0].findVariable(var);
-                return new ColumnAccessEvalFactory(index);
-            }
-        }
-        throw new IllegalArgumentException("Unknown expression type: " + expr.getExpressionTag());
-    }
-
-    @Override
-    public ICopyAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
-            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
-            JobGenContext context) throws AlgebricksException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ICopyRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public ICopyUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException {
-        throw new UnsupportedOperationException();
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
deleted file mode 100644
index c52a4bd..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-
-public interface IPigletFunctionEvaluatorFactoryBuilder {
-    public ICopyEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] arguments);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
deleted file mode 100644
index e84964d..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class IntegerEqFunctionEvaluatorFactory implements ICopyEvaluatorFactory {
-    private static final long serialVersionUID = 1L;
-
-    private final ICopyEvaluatorFactory arg1Factory;
-
-    private final ICopyEvaluatorFactory arg2Factory;
-
-    public IntegerEqFunctionEvaluatorFactory(ICopyEvaluatorFactory arg1Factory, ICopyEvaluatorFactory arg2Factory) {
-        this.arg1Factory = arg1Factory;
-        this.arg2Factory = arg2Factory;
-    }
-
-    @Override
-    public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
-        return new ICopyEvaluator() {
-            private DataOutput dataout = output.getDataOutput();
-            private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
-            private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
-            private ICopyEvaluator eval1 = arg1Factory.createEvaluator(out1);
-            private ICopyEvaluator eval2 = arg2Factory.createEvaluator(out2);
-
-            @Override
-            public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
-                out1.reset();
-                eval1.evaluate(tuple);
-                out2.reset();
-                eval2.evaluate(tuple);
-                int v1 = IntegerPointable.getInteger(out1.getByteArray(), 0);
-                int v2 = IntegerPointable.getInteger(out2.getByteArray(), 0);
-                boolean r = v1 == v2;
-                try {
-                    dataout.writeBoolean(r);
-                } catch (IOException ioe) {
-                    throw new AlgebricksException(ioe);
-                }
-            }
-        };
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
deleted file mode 100644
index 35015a5..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-
-public class PigletFunctionRegistry {
-    private static final Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> builderMap;
-
-    static {
-        Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> temp = new HashMap<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder>();
-
-        temp.put(AlgebricksBuiltinFunctions.EQ, new IPigletFunctionEvaluatorFactoryBuilder() {
-            @Override
-            public ICopyEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] arguments) {
-                return new IntegerEqFunctionEvaluatorFactory(arguments[0], arguments[1]);
-            }
-        });
-
-        builderMap = Collections.unmodifiableMap(temp);
-    }
-
-    public static ICopyEvaluatorFactory createFunctionEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] args)
-            throws PigletException {
-        IPigletFunctionEvaluatorFactoryBuilder builder = builderMap.get(fid);
-        if (builder == null) {
-            throw new PigletException("Unknown function: " + fid);
-        }
-        return builder.buildEvaluatorFactory(fid, args);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java
deleted file mode 100644
index 9aec8bb..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class BagType extends Type {
-    @Override
-    public Tag getTag() {
-        return Tag.BAG;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java
deleted file mode 100644
index 4bef8ff..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class CharArrayType extends Type {
-    public static final Type INSTANCE = new CharArrayType();
-
-    private CharArrayType() {
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.CHAR_ARRAY;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java
deleted file mode 100644
index 8b7a0bd..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class DoubleType extends Type {
-    public static final Type INSTANCE = new DoubleType();
-
-    private DoubleType() {
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.DOUBLE;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java
deleted file mode 100644
index 69f5db6..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class FloatType extends Type {
-    public static final Type INSTANCE = new FloatType();
-
-    private FloatType() {
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.FLOAT;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java
deleted file mode 100644
index 9c568df..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class IntegerType extends Type {
-    public static final Type INSTANCE = new IntegerType();
-
-    private IntegerType() {
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.INTEGER;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java
deleted file mode 100644
index b992afc..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class LongType extends Type {
-    public static final Type INSTANCE = new LongType();
-
-    private LongType() {
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.LONG;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java
deleted file mode 100644
index a46e40d..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class MapType extends Type {
-    @Override
-    public Tag getTag() {
-        return Tag.MAP;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
deleted file mode 100644
index 1702353..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-
-public class Schema {
-    private List<Pair<String, Type>> schema;
-
-    public Schema(List<Pair<String, Type>> schema) {
-        this.schema = schema;
-    }
-
-    public List<Pair<String, Type>> getSchema() {
-        return schema;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java
deleted file mode 100644
index 429026f..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public class TupleType extends Type {
-    @Override
-    public Tag getTag() {
-        return Tag.TUPLE;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java
deleted file mode 100644
index 36efcea..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
-
-public abstract class Type {
-    public enum Tag {
-        INTEGER,
-        LONG,
-        FLOAT,
-        DOUBLE,
-        CHAR_ARRAY,
-        TUPLE,
-        BAG,
-        MAP
-    }
-    
-    public abstract Tag getTag();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ASTNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ASTNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ASTNode.java
new file mode 100644
index 0000000..1bdf6b2
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ASTNode.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public abstract class ASTNode {
+    public enum Tag {
+        ASSIGNMENT,
+        DUMP,
+        LOAD,
+        FILTER,
+
+        SCALAR_FUNCTION,
+        LITERAL,
+        FIELD_ACCESS,
+    }
+
+    public abstract Tag getTag();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
new file mode 100644
index 0000000..71cf26d
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class AssignmentNode extends ASTNode {
+    private String alias;
+
+    private RelationNode relation;
+
+    public AssignmentNode(String alias, RelationNode relation) {
+        this.alias = alias;
+        this.relation = relation;
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.ASSIGNMENT;
+    }
+
+    public String getAlias() {
+        return alias;
+    }
+
+    public RelationNode getRelation() {
+        return relation;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/DumpNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/DumpNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/DumpNode.java
new file mode 100644
index 0000000..42fca63
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/DumpNode.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class DumpNode extends RelationNode {
+    private final String file;
+    private final String alias;
+
+    public DumpNode(String file, String alias) {
+        this.file = file;
+        this.alias = alias;
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.DUMP;
+    }
+
+    public String getFile() {
+        return file;
+    }
+
+    public String getAlias() {
+        return alias;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
new file mode 100644
index 0000000..c1482c2
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public abstract class ExpressionNode extends ASTNode {
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
new file mode 100644
index 0000000..90b54e6
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class FieldAccessExpressionNode extends ExpressionNode {
+    private String relationName;
+
+    private String fieldName;
+
+    public FieldAccessExpressionNode(String relationName, String fieldName) {
+        this.relationName = relationName;
+        this.fieldName = fieldName;
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.FIELD_ACCESS;
+    }
+
+    public String getRelationName() {
+        return relationName;
+    }
+
+    public String getFieldName() {
+        return fieldName;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FilterNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FilterNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FilterNode.java
new file mode 100644
index 0000000..b917cbb
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FilterNode.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class FilterNode extends RelationNode {
+    private String alias;
+
+    private ExpressionNode expression;
+
+    public FilterNode(String alias, ExpressionNode expression) {
+        this.alias = alias;
+        this.expression = expression;
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.FILTER;
+    }
+
+    public String getAlias() {
+        return alias;
+    }
+
+    public ExpressionNode getExpression() {
+        return expression;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FunctionTag.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
new file mode 100644
index 0000000..523efb1
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public enum FunctionTag {
+    BOOLEAN_AND,
+    BOOLEAN_OR,
+    BOOLEAN_NOT,
+
+    EQ,
+    NEQ,
+    LT,
+    LTE,
+    GT,
+    GTE,
+
+    ADD,
+    SUBTRACT,
+    MULTIPLY,
+    DIVIDE,
+    MOD,
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
new file mode 100644
index 0000000..7b3f2bd
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+
+public class LiteralExpressionNode extends ExpressionNode {
+    private String image;
+
+    private Type type;
+
+    public LiteralExpressionNode(String image, Type type) {
+        this.image = image;
+        this.type = type;
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.LITERAL;
+    }
+
+    public String getImage() {
+        return image;
+    }
+
+    public Type getType() {
+        return type;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LoadNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LoadNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LoadNode.java
new file mode 100644
index 0000000..0ff16b9
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/LoadNode.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
+
+public class LoadNode extends RelationNode {
+    private String dataFile;
+
+    private Schema schema;
+
+    public LoadNode(String dataFile, Schema schema) {
+        this.dataFile = dataFile;
+        this.schema = schema;
+    }
+    
+    @Override
+    public Tag getTag() {
+        return Tag.LOAD;
+    }
+
+    public String getDataFile() {
+        return dataFile;
+    }
+
+    public Schema getSchema() {
+        return schema;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/RelationNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/RelationNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/RelationNode.java
new file mode 100644
index 0000000..9bd7324
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/RelationNode.java
@@ -0,0 +1,18 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public abstract class RelationNode extends ASTNode {
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
new file mode 100644
index 0000000..a0ce89b
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+import java.util.List;
+
+public class ScalarFunctionExpressionNode extends ExpressionNode {
+    private FunctionTag fTag;
+
+    private String fName;
+
+    private List<ASTNode> arguments;
+
+    public ScalarFunctionExpressionNode(FunctionTag fTag, String fName, List<ASTNode> arguments) {
+        this.fTag = fTag;
+        this.fName = fName;
+        this.arguments = arguments;
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.SCALAR_FUNCTION;
+    }
+
+    public FunctionTag getFunctionTag() {
+        return fTag;
+    }
+
+    public String getFunctionName() {
+        return fName;
+    }
+
+    public List<ASTNode> getArguments() {
+        return arguments;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
new file mode 100644
index 0000000..82276d0
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+
+public final class ConstantValue implements IAlgebricksConstantValue {
+    private final Type type;
+
+    private final String image;
+
+    public ConstantValue(Type type, String image) {
+        this.type = type;
+        this.image = image;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public String getImage() {
+        return image;
+    }
+
+    @Override
+    public boolean isFalse() {
+        return false;
+    }
+
+    @Override
+    public boolean isNull() {
+        return false;
+    }
+
+    @Override
+    public boolean isTrue() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
new file mode 100644
index 0000000..820d016
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
@@ -0,0 +1,376 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
+
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
+import edu.uci.ics.hyracks.algebricks.compiler.api.ICompiler;
+import edu.uci.ics.hyracks.algebricks.compiler.api.ICompilerFactory;
+import edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController;
+import edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.LogicalExpressionJobGenToExpressionRuntimeProviderAdapter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.AssignmentNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.DumpNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FieldAccessExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FilterNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FunctionTag;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.LiteralExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.LoadNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.RelationNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ScalarFunctionExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletFileDataSink;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletFileDataSource;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.parser.ParseException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.parser.PigletParser;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.rewriter.PigletRewriteRuleset;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.PigletExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class PigletCompiler {
+    private static final Logger LOGGER = Logger.getLogger(PigletCompiler.class.getName());
+
+    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultLogicalRewrites() {
+        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
+        SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
+        SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
+        SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
+                PigletRewriteRuleset.buildTypeInferenceRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlFullDfs,
+                PigletRewriteRuleset.buildNormalizationRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+                PigletRewriteRuleset.buildCondPushDownRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+                PigletRewriteRuleset.buildJoinInferenceRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+                PigletRewriteRuleset.buildOpPushDownRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
+                PigletRewriteRuleset.buildDataExchangeRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+                PigletRewriteRuleset.buildConsolidationRuleCollection()));
+        return defaultLogicalRewrites;
+    }
+
+    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultPhysicalRewrites() {
+        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
+        SequentialOnceRuleController seqOnceCtrlAllLevels = new SequentialOnceRuleController(true);
+        SequentialOnceRuleController seqOnceCtrlTopLevel = new SequentialOnceRuleController(false);
+        defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels,
+                PigletRewriteRuleset.buildPhysicalRewritesAllLevelsRuleCollection()));
+        defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlTopLevel,
+                PigletRewriteRuleset.buildPhysicalRewritesTopLevelRuleCollection()));
+        defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels,
+                PigletRewriteRuleset.prepareForJobGenRuleCollection()));
+        return defaultPhysicalRewrites;
+    }
+
+    private final ICompilerFactory cFactory;
+
+    private final PigletMetadataProvider metadataProvider;
+
+    private int varCounter;
+
+    private ILogicalOperator previousOp;
+
+    public PigletCompiler() {
+        HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder();
+        builder.setLogicalRewrites(buildDefaultLogicalRewrites());
+        builder.setPhysicalRewrites(buildDefaultPhysicalRewrites());
+        builder.setSerializerDeserializerProvider(new ISerializerDeserializerProvider() {
+            @SuppressWarnings("unchecked")
+            @Override
+            public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException {
+                return null;
+            }
+        });
+        builder.setTypeTraitProvider(new ITypeTraitProvider() {
+            public ITypeTraits getTypeTrait(Object type) {
+                return null;
+            }
+        });
+        builder.setPrinterProvider(PigletPrinterFactoryProvider.INSTANCE);
+        builder.setExpressionRuntimeProvider(new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(
+                new PigletExpressionJobGen()));
+        builder.setExpressionTypeComputer(new IExpressionTypeComputer() {
+            @Override
+            public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
+                    IVariableTypeEnvironment env) throws AlgebricksException {
+                return null;
+            }
+        });
+        cFactory = builder.create();
+        metadataProvider = new PigletMetadataProvider();
+    }
+
+    public List<ASTNode> parse(Reader in) throws ParseException {
+        PigletParser parser = new PigletParser(in);
+        List<ASTNode> statements = parser.Statements();
+        return statements;
+    }
+
+    public JobSpecification compile(List<ASTNode> ast) throws AlgebricksException, PigletException {
+        ILogicalPlan plan = translate(ast);
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Translated Plan:");
+            LOGGER.info(getPrettyPrintedPlan(plan));
+        }
+        ICompiler compiler = cFactory.createCompiler(plan, metadataProvider, varCounter);
+        compiler.optimize();
+        if (LOGGER.isLoggable(Level.INFO)) {
+            LOGGER.info("Optimized Plan:");
+            LOGGER.info(getPrettyPrintedPlan(plan));
+        }
+        return compiler.createJob(null, null);
+    }
+
+    private ILogicalPlan translate(List<ASTNode> ast) throws PigletException {
+        Map<String, Relation> symMap = new HashMap<String, Relation>();
+        List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
+        previousOp = null;
+        for (ASTNode an : ast) {
+            switch (an.getTag()) {
+                case DUMP: {
+                    DumpNode dn = (DumpNode) an;
+                    Relation input = symMap.get(dn.getAlias());
+                    List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
+                    for (LogicalVariable v : input.schema.values()) {
+                        expressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v)));
+                    }
+                    PigletFileDataSink dataSink = new PigletFileDataSink(dn.getFile());
+                    ILogicalOperator op = new WriteOperator(expressions, dataSink);
+                    op.getInputs().add(new MutableObject<ILogicalOperator>(input.op));
+                    roots.add(new MutableObject<ILogicalOperator>(op));
+                }
+                    break;
+
+                case ASSIGNMENT: {
+                    AssignmentNode asn = (AssignmentNode) an;
+                    String alias = asn.getAlias();
+                    RelationNode rn = asn.getRelation();
+                    Relation rel = translate(rn, symMap);
+                    previousOp = rel.op;
+                    rel.alias = alias;
+                    symMap.put(alias, rel);
+                }
+                    break;
+            }
+        }
+        return new ALogicalPlanImpl(roots);
+    }
+
+    private Relation translate(RelationNode rn, Map<String, Relation> symMap) throws PigletException {
+        switch (rn.getTag()) {
+            case LOAD: {
+                LoadNode ln = (LoadNode) rn;
+                String file = ln.getDataFile();
+                Schema schema = ln.getSchema();
+                List<Pair<String, Type>> fieldsSchema = schema.getSchema();
+                List<LogicalVariable> variables = new ArrayList<LogicalVariable>();
+                List<Object> types = new ArrayList<Object>();
+                Relation rel = new Relation();
+                for (Pair<String, Type> p : fieldsSchema) {
+                    LogicalVariable v = newVariable();
+                    rel.schema.put(p.first, v);
+                    variables.add(v);
+                    types.add(p.second);
+                }
+                PigletFileDataSource ds = new PigletFileDataSource(file, types.toArray());
+                rel.op = new DataSourceScanOperator(variables, ds);
+                rel.op.getInputs().add(
+                        new MutableObject<ILogicalOperator>(previousOp == null ? new EmptyTupleSourceOperator()
+                                : previousOp));
+                return rel;
+            }
+
+            case FILTER: {
+                FilterNode fn = (FilterNode) rn;
+                String alias = fn.getAlias();
+                ExpressionNode conditionNode = fn.getExpression();
+                Relation inputRel = findInputRelation(alias, symMap);
+                Pair<Relation, LogicalVariable> tempInput = translateScalarExpression(inputRel, conditionNode);
+                Relation rel = new Relation();
+                rel.op = new SelectOperator(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+                        tempInput.second)), false, null);
+                rel.op.getInputs().add(new MutableObject<ILogicalOperator>(tempInput.first.op));
+                rel.schema.putAll(tempInput.first.schema);
+                return rel;
+            }
+        }
+        throw new IllegalArgumentException("Unknown node: " + rn.getTag() + " encountered");
+    }
+
+    private Pair<Relation, LogicalVariable> translateScalarExpression(Relation inputRel, ExpressionNode expressionNode)
+            throws PigletException {
+        switch (expressionNode.getTag()) {
+            case FIELD_ACCESS: {
+                FieldAccessExpressionNode faen = (FieldAccessExpressionNode) expressionNode;
+                String fieldName = faen.getFieldName();
+                LogicalVariable lVar = findField(fieldName, inputRel.schema);
+                return new Pair<Relation, LogicalVariable>(inputRel, lVar);
+            }
+
+            case LITERAL: {
+                LiteralExpressionNode len = (LiteralExpressionNode) expressionNode;
+                String image = len.getImage();
+                Type type = len.getType();
+                ConstantExpression ce = new ConstantExpression(new ConstantValue(type, image));
+                Relation rel = new Relation();
+                LogicalVariable var = newVariable();
+                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+                vars.add(var);
+
+                List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+                exprs.add(new MutableObject<ILogicalExpression>(ce));
+
+                rel.op = new AssignOperator(vars, exprs);
+                rel.op.getInputs().add(new MutableObject<ILogicalOperator>(inputRel.op));
+                rel.schema.putAll(inputRel.schema);
+
+                return new Pair<Relation, LogicalVariable>(rel, var);
+            }
+
+            case SCALAR_FUNCTION: {
+                ScalarFunctionExpressionNode sfen = (ScalarFunctionExpressionNode) expressionNode;
+                List<Mutable<ILogicalExpression>> argExprs = new ArrayList<Mutable<ILogicalExpression>>();
+                List<ASTNode> arguments = sfen.getArguments();
+                Relation rel = inputRel;
+                for (ASTNode a : arguments) {
+                    Pair<Relation, LogicalVariable> argPair = translateScalarExpression(rel, (ExpressionNode) a);
+                    rel = argPair.first;
+                    argExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(argPair.second)));
+                }
+                Relation outRel = new Relation();
+                outRel.schema.putAll(rel.schema);
+                LogicalVariable var = newVariable();
+                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+                vars.add(var);
+
+                IFunctionInfo fInfo = lookupFunction(sfen.getFunctionTag(), sfen.getFunctionName());
+
+                List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+                exprs.add(new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(fInfo, argExprs)));
+                outRel.op = new AssignOperator(vars, exprs);
+                outRel.op.getInputs().add(new MutableObject<ILogicalOperator>(rel.op));
+                return new Pair<Relation, LogicalVariable>(outRel, var);
+            }
+        }
+        return null;
+    }
+
+    private IFunctionInfo lookupFunction(FunctionTag functionTag, String functionName) throws PigletException {
+        switch (functionTag) {
+            case EQ:
+                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.EQ);
+
+            case NEQ:
+                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.NEQ);
+
+            case LT:
+                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.LT);
+
+            case LTE:
+                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.LE);
+
+            case GT:
+                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.GT);
+
+            case GTE:
+                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.GE);
+        }
+        throw new PigletException("Unsupported function: " + functionTag);
+    }
+
+    private LogicalVariable newVariable() {
+        return new LogicalVariable(varCounter++);
+    }
+
+    private LogicalVariable findField(String fieldName, Map<String, LogicalVariable> schema) throws PigletException {
+        LogicalVariable var = schema.get(fieldName);
+        if (var == null) {
+            throw new PigletException("Unable to find field named: " + fieldName);
+        }
+        return var;
+    }
+
+    private Relation findInputRelation(String alias, Map<String, Relation> symMap) throws PigletException {
+        Relation rel = symMap.get(alias);
+        if (rel == null) {
+            throw new PigletException("Unknown alias " + alias + "referenced");
+        }
+        return rel;
+    }
+
+    private static class Relation {
+        String alias;
+        ILogicalOperator op;
+        final Map<String, LogicalVariable> schema;
+
+        public Relation() {
+            schema = new LinkedHashMap<String, LogicalVariable>();
+        }
+    }
+
+    private String getPrettyPrintedPlan(ILogicalPlan plan) throws AlgebricksException {
+        LogicalOperatorPrettyPrintVisitor v = new LogicalOperatorPrettyPrintVisitor();
+        StringBuilder buffer = new StringBuilder();
+        PlanPrettyPrinter.printPlan(plan, buffer, v, 0);
+        return buffer.toString();
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
new file mode 100644
index 0000000..beb4e32
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.impl.IntegerPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.data.std.primitive.FloatPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+
+public class PigletPrinterFactoryProvider implements IPrinterFactoryProvider {
+
+    public static final PigletPrinterFactoryProvider INSTANCE = new PigletPrinterFactoryProvider();
+
+    private PigletPrinterFactoryProvider() {
+    }
+
+    @Override
+    public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException {
+        Type t = (Type) type;
+        switch (t.getTag()) {
+            case INTEGER:
+                return IntegerPrinterFactory.INSTANCE;
+            case CHAR_ARRAY:
+                return CharArrayPrinterFactory.INSTANCE;
+            case FLOAT:
+                return FloatPrinterFactory.INSTANCE;
+            default:
+                throw new UnsupportedOperationException();
+
+        }
+    }
+
+    public static class CharArrayPrinterFactory implements IPrinterFactory {
+
+        private static final long serialVersionUID = 1L;
+
+        public static final CharArrayPrinterFactory INSTANCE = new CharArrayPrinterFactory();
+
+        private CharArrayPrinterFactory() {
+        }
+
+        @Override
+        public IPrinter createPrinter() {
+            return new IPrinter() {
+                @Override
+                public void init() throws AlgebricksException {
+                }
+
+                @Override
+                public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+                    try {
+                        WriteValueTools.writeUTF8String(b, s, l, ps);
+                    } catch (IOException e) {
+                        throw new AlgebricksException(e);
+                    }
+                }
+            };
+        }
+    }
+
+    public static class FloatPrinterFactory implements IPrinterFactory {
+
+        private static final long serialVersionUID = 1L;
+
+        public static final FloatPrinterFactory INSTANCE = new FloatPrinterFactory();
+
+        private FloatPrinterFactory() {
+        }
+
+        @Override
+        public IPrinter createPrinter() {
+            return new IPrinter() {
+                @Override
+                public void init() throws AlgebricksException {
+                }
+
+                @Override
+                public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+                    ps.print(FloatPointable.getFloat(b, s));
+                }
+            };
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/exceptions/PigletException.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
new file mode 100644
index 0000000..aa28f34
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions;
+
+public class PigletException extends Exception {
+    private static final long serialVersionUID = 1L;
+
+    public PigletException(String message) {
+        super(message);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
new file mode 100644
index 0000000..1ac4259
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class FileSplitUtils {
+    public static FileSplit[] parseFileSplits(String fileSplits) {
+        String[] splits = fileSplits.split(",");
+        FileSplit[] fSplits = new FileSplit[splits.length];
+        for (int i = 0; i < splits.length; ++i) {
+            String s = splits[i].trim();
+            int idx = s.indexOf(':');
+            if (idx < 0) {
+                throw new IllegalArgumentException("File split " + s + " not well formed");
+            }
+            fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1))));
+        }
+        return fSplits;
+    }
+}
\ No newline at end of file

[29/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
new file mode 100644
index 0000000..99df45c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
@@ -0,0 +1,447 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class IsomorphismVariableMappingVisitor implements ILogicalOperatorVisitor<Void, ILogicalOperator> {
+
+    private Map<LogicalVariable, LogicalVariable> variableMapping;
+
+    public IsomorphismVariableMappingVisitor(Map<LogicalVariable, LogicalVariable> variableMapping) {
+        this.variableMapping = variableMapping;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariablesForAbstractAssign(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariablesForAbstractAssign(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariablesForGroupBy(op, arg);
+        mapVariablesInNestedPlans(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        ILogicalOperator inputToCreator1 = op.getSourceOperator();
+        NestedTupleSourceOperator nts = (NestedTupleSourceOperator) arg;
+        ILogicalOperator inputToCreator2 = nts.getSourceOperator();
+        inputToCreator1.accept(this, inputToCreator2);
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariablesForAbstractAssign(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariablesInNestedPlans(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariablesForUnion(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    private void mapChildren(ILogicalOperator op, ILogicalOperator opArg) throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> inputs = op.getInputs();
+        List<Mutable<ILogicalOperator>> inputsArg = opArg.getInputs();
+        if (inputs.size() != inputsArg.size())
+            throw new AlgebricksException("children are not isomoprhic");
+        for (int i = 0; i < inputs.size(); i++) {
+            ILogicalOperator input = inputs.get(i).getValue();
+            ILogicalOperator inputArg = inputsArg.get(i).getValue();
+            input.accept(this, inputArg);
+        }
+    }
+
+    private void mapVariables(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
+        List<LogicalVariable> producedVarLeft = new ArrayList<LogicalVariable>();
+        List<LogicalVariable> producedVarRight = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(left, producedVarLeft);
+        VariableUtilities.getProducedVariables(right, producedVarRight);
+        mapVariables(producedVarLeft, producedVarRight);
+    }
+
+    private void mapVariables(List<LogicalVariable> variablesLeft, List<LogicalVariable> variablesRight) {
+        if (variablesLeft.size() != variablesRight.size())
+            return;
+        int size = variablesLeft.size();
+        for (int i = 0; i < size; i++) {
+            LogicalVariable left = variablesLeft.get(i);
+            LogicalVariable right = variablesRight.get(i);
+            variableMapping.put(right, left);
+        }
+    }
+
+    private void mapVariablesForAbstractAssign(ILogicalOperator left, ILogicalOperator right)
+            throws AlgebricksException {
+        AbstractAssignOperator leftOp = (AbstractAssignOperator) left;
+        AbstractAssignOperator rightOp = (AbstractAssignOperator) right;
+        List<LogicalVariable> producedVarLeft = new ArrayList<LogicalVariable>();
+        List<LogicalVariable> producedVarRight = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(left, producedVarLeft);
+        VariableUtilities.getProducedVariables(right, producedVarRight);
+        mapVariablesForAbstractAssign(producedVarLeft, leftOp.getExpressions(), producedVarRight,
+                rightOp.getExpressions());
+    }
+
+    private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
+        GroupByOperator leftOp = (GroupByOperator) left;
+        GroupByOperator rightOp = (GroupByOperator) right;
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList();
+        mapVarExprPairList(leftPairs, rightPairs);
+        leftPairs = leftOp.getDecorList();
+        rightPairs = rightOp.getDecorList();
+        mapVarExprPairList(leftPairs, rightPairs);
+    }
+
+    private void mapVarExprPairList(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs) {
+        if (leftPairs.size() != rightPairs.size())
+            return;
+        for (int i = 0; i < leftPairs.size(); i++) {
+            ILogicalExpression exprLeft = leftPairs.get(i).second.getValue();
+            LogicalVariable leftVar = leftPairs.get(i).first;
+            for (int j = 0; j < leftPairs.size(); j++) {
+                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(rightPairs.get(j).second).getValue();
+                if (exprLeft.equals(exprRight)) {
+                    LogicalVariable rightVar = rightPairs.get(j).first;
+                    if (rightVar != null && leftVar != null) {
+                        variableMapping.put(rightVar, leftVar);
+                    }
+                    break;
+                }
+            }
+        }
+    }
+
+    private void mapVariablesForAbstractAssign(List<LogicalVariable> variablesLeft,
+            List<Mutable<ILogicalExpression>> exprsLeft, List<LogicalVariable> variablesRight,
+            List<Mutable<ILogicalExpression>> exprsRight) {
+        if (variablesLeft.size() != variablesRight.size())
+            return;
+        int size = variablesLeft.size();
+        for (int i = 0; i < size; i++) {
+            ILogicalExpression exprLeft = exprsLeft.get(i).getValue();
+            LogicalVariable left = variablesLeft.get(i);
+            for (int j = 0; j < size; j++) {
+                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(exprsRight.get(j)).getValue();
+                if (exprLeft.equals(exprRight)) {
+                    LogicalVariable right = variablesRight.get(j);
+                    variableMapping.put(right, left);
+                    break;
+                }
+            }
+        }
+    }
+
+    private void mapVariablesInNestedPlans(ILogicalOperator opOrigin, ILogicalOperator arg) throws AlgebricksException {
+        AbstractOperatorWithNestedPlans op = (AbstractOperatorWithNestedPlans) opOrigin;
+        AbstractOperatorWithNestedPlans argOp = (AbstractOperatorWithNestedPlans) arg;
+        List<ILogicalPlan> plans = op.getNestedPlans();
+        List<ILogicalPlan> plansArg = argOp.getNestedPlans();
+        if (plans.size() != plansArg.size())
+            return;
+        for (int i = 0; i < plans.size(); i++) {
+            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
+            if (roots.size() != rootsArg.size())
+                return;
+            for (int j = 0; j < roots.size(); j++) {
+                ILogicalOperator topOp1 = roots.get(j).getValue();
+                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
+                topOp1.accept(this, topOp2);
+            }
+        }
+    }
+
+    private void mapVariablesStandard(ILogicalOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapChildren(op, arg);
+        mapVariables(op, arg);
+    }
+
+    private Mutable<ILogicalExpression> copyExpressionAndSubtituteVars(Mutable<ILogicalExpression> expr) {
+        ILogicalExpression copy = ((AbstractLogicalExpression) expr.getValue()).cloneExpression();
+        for (Entry<LogicalVariable, LogicalVariable> entry : variableMapping.entrySet())
+            copy.substituteVar(entry.getKey(), entry.getValue());
+        return new MutableObject<ILogicalExpression>(copy);
+    }
+
+    private void mapVariablesForUnion(ILogicalOperator op, ILogicalOperator arg) {
+        UnionAllOperator union = (UnionAllOperator) op;
+        UnionAllOperator unionArg = (UnionAllOperator) arg;
+        mapVarTripleList(union.getVariableMappings(), unionArg.getVariableMappings());
+    }
+
+    private void mapVarTripleList(List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> leftTriples,
+            List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> rightTriples) {
+        if (leftTriples.size() != rightTriples.size())
+            return;
+        for (int i = 0; i < leftTriples.size(); i++) {
+            LogicalVariable leftFirstVar = leftTriples.get(i).first;
+            LogicalVariable leftSecondVar = leftTriples.get(i).second;
+            LogicalVariable leftThirdVar = leftTriples.get(i).third;
+            for (int j = 0; j < rightTriples.size(); j++) {
+                LogicalVariable rightFirstVar = rightTriples.get(j).first;
+                LogicalVariable rightSecondVar = rightTriples.get(j).second;
+                LogicalVariable rightThirdVar = rightTriples.get(j).third;
+                if (varEquivalent(leftFirstVar, rightFirstVar) && varEquivalent(leftSecondVar, rightSecondVar)) {
+                    variableMapping.put(rightThirdVar, leftThirdVar);
+                    break;
+                }
+            }
+        }
+    }
+
+    private boolean varEquivalent(LogicalVariable left, LogicalVariable right) {
+        if (variableMapping.get(right) == null)
+            return false;
+        if (variableMapping.get(right).equals(left))
+            return true;
+        else
+            return false;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, ILogicalOperator arg) throws AlgebricksException {
+        mapVariablesStandard(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, ILogicalOperator arg)
+            throws AlgebricksException {
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
new file mode 100644
index 0000000..19fcf65
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
@@ -0,0 +1,350 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LogicalPropertiesVectorImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class LogicalPropertiesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
+
+    public static void computeLogicalPropertiesDFS(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        LogicalPropertiesVisitor visitor = new LogicalPropertiesVisitor();
+        computeLogicalPropertiesRec(op, visitor, context);
+    }
+
+    private static void computeLogicalPropertiesRec(ILogicalOperator op, LogicalPropertiesVisitor visitor,
+            IOptimizationContext context) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> ref : op.getInputs()) {
+            computeLogicalPropertiesRec(ref.getValue(), visitor, context);
+        }
+        op.accept(visitor, context);
+        if (AlgebricksConfig.DEBUG) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.finest("Logical properties visitor for " + op + ": "
+                    + context.getLogicalPropertiesVector(op) + "\n");
+        }
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, IOptimizationContext context) throws AlgebricksException {
+        visitAssignment(op, context);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, IOptimizationContext arg) throws AlgebricksException {
+        propagateCardinalityAndFrameNumber(op, arg);
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, IOptimizationContext arg) throws AlgebricksException {
+        Object annot1 = op.getAnnotations().get(OperatorAnnotations.CARDINALITY);
+        if (annot1 == null) {
+            return null;
+        }
+        Integer m = (Integer) annot1;
+        LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl();
+        v.setNumberOfTuples(m);
+        Object annot2 = op.getAnnotations().get(OperatorAnnotations.MAX_NUMBER_FRAMES);
+        if (annot2 != null) {
+            Integer f = (Integer) annot2;
+            v.setMaxOutputFrames(f);
+        }
+        arg.putLogicalPropertiesVector(op, v);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, IOptimizationContext context) throws AlgebricksException {
+        propagateCardinalityAndFrameNumber(op, context);
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        visitAssignment(op, context);
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    private LogicalPropertiesVectorImpl propagateCardinality(ILogicalOperator op, IOptimizationContext context) {
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
+        ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
+        if (v0 == null) {
+            return null;
+        }
+        LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl();
+        v.setNumberOfTuples(v0.getNumberOfTuples());
+        context.putLogicalPropertiesVector(op, v);
+        return v;
+    }
+
+    private void visitAssignment(AbstractAssignOperator op, IOptimizationContext context) throws AlgebricksException {
+        LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
+        if (v != null && v.getNumberOfTuples() != null) {
+            IVariableEvalSizeEnvironment varSizeEnv = context.getVariableEvalSizeEnvironment();
+            IExpressionEvalSizeComputer evalSize = context.getExpressionEvalSizeComputer();
+            if (evalSize != null) {
+                ILogicalOperator op0 = op.getInputs().get(0).getValue();
+                ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
+                if (v0 != null) {
+                    long frames0 = v0.getMaxOutputFrames();
+                    long overhead = 0; // added per tuple
+                    for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+                        int sz = evalSize.getEvalSize(exprRef.getValue(), varSizeEnv);
+                        if (sz == -1) {
+                            return;
+                        }
+                        overhead += sz;
+                    }
+                    int frameSize = context.getPhysicalOptimizationConfig().getFrameSize();
+                    if (frameSize > 0) {
+                        long sz = frames0 * frameSize + overhead * v.getNumberOfTuples();
+                        int frames1 = (int) (sz / frameSize);
+                        if (sz % frameSize > 0) {
+                            frames1++;
+                        }
+                        v.setMaxOutputFrames(frames1);
+                    }
+                }
+            }
+        }
+    }
+
+    public void propagateCardinalityAndFrameNumber(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
+        // propagate also max number of frames (conservatively)
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
+        ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
+        if (v0 != null) {
+            v.setMaxOutputFrames(v0.getMaxOutputFrames());
+        }
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, IOptimizationContext arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, IOptimizationContext arg)
+            throws AlgebricksException {
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
new file mode 100644
index 0000000..e62dbf8
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
@@ -0,0 +1,368 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class OperatorDeepCopyVisitor implements ILogicalOperatorVisitor<ILogicalOperator, Void> {
+
+    @Override
+    public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        newList.addAll(op.getVariables());
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new AggregateOperator(newList, newExpressions);
+    }
+
+    @Override
+    public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
+            throws AlgebricksException {
+        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        newList.addAll(op.getVariables());
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new RunningAggregateOperator(newList, newExpressions);
+    }
+
+    @Override
+    public ILogicalOperator visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg)
+            throws AlgebricksException {
+        return new EmptyTupleSourceOperator();
+    }
+
+    @Override
+    public ILogicalOperator visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decoList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+        ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getGroupByList())
+            groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
+                    deepCopyExpressionRef(pair.second)));
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList())
+            decoList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
+                    deepCopyExpressionRef(pair.second)));
+        for (ILogicalPlan plan : op.getNestedPlans()) {
+            newSubplans.add(OperatorManipulationUtil.deepCopy(plan));
+        }
+        return new GroupByOperator(groupByList, decoList, newSubplans);
+    }
+
+    @Override
+    public ILogicalOperator visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+        return new LimitOperator(deepCopyExpressionRef(op.getMaxObjects()).getValue(), deepCopyExpressionRef(
+                op.getOffset()).getValue(), op.isTopmostLimitOp());
+    }
+
+    @Override
+    public ILogicalOperator visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+        return new InnerJoinOperator(deepCopyExpressionRef(op.getCondition()), op.getInputs().get(0), op.getInputs()
+                .get(1));
+    }
+
+    @Override
+    public ILogicalOperator visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+        return new LeftOuterJoinOperator(deepCopyExpressionRef(op.getCondition()), op.getInputs().get(0), op
+                .getInputs().get(1));
+    }
+
+    @Override
+    public ILogicalOperator visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg)
+            throws AlgebricksException {
+        return new NestedTupleSourceOperator(null);
+    }
+
+    @Override
+    public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+        return new OrderOperator(this.deepCopyOrderAndExpression(op.getOrderExpressions()));
+    }
+
+    @Override
+    public ILogicalOperator visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        newList.addAll(op.getVariables());
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new AssignOperator(newList, newExpressions);
+    }
+
+    @Override
+    public ILogicalOperator visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+        return new SelectOperator(deepCopyExpressionRef(op.getCondition()), op.getRetainNull(),
+                op.getNullPlaceholderVariable());
+    }
+
+    @Override
+    public ILogicalOperator visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+        newList.addAll(op.getVariables());
+        return new ProjectOperator(newList);
+    }
+
+    @Override
+    public ILogicalOperator visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg)
+            throws AlgebricksException {
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new PartitioningSplitOperator(newExpressions, op.getDefaultBranchIndex());
+    }
+
+    @Override
+    public ILogicalOperator visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        return new ReplicateOperator(op.getOutputArity());
+    }
+
+    @Override
+    public ILogicalOperator visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+        ArrayList<LogicalVariable> newOutputList = new ArrayList<LogicalVariable>();
+        newInputList.addAll(op.getInputVariables());
+        newOutputList.addAll(op.getOutputVariables());
+        return new ScriptOperator(op.getScriptDescription(), newInputList, newOutputList);
+    }
+
+    @Override
+    public ILogicalOperator visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
+        for (ILogicalPlan plan : op.getNestedPlans()) {
+            newSubplans.add(OperatorManipulationUtil.deepCopy(plan));
+        }
+        return new SubplanOperator(newSubplans);
+    }
+
+    @Override
+    public ILogicalOperator visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> newVarMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>();
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple : varMap)
+            newVarMap.add(new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(triple.first, triple.second,
+                    triple.third));
+        return new UnionAllOperator(newVarMap);
+    }
+
+    @Override
+    public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+        return new UnnestOperator(op.getVariable(), deepCopyExpressionRef(op.getExpressionRef()),
+                op.getPositionalVariable(), op.getPositionalVariableType(), op.getPositionWriter());
+    }
+
+    @Override
+    public ILogicalOperator visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+        newInputList.addAll(op.getVariables());
+        return new UnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()), new ArrayList<Object>(
+                op.getVariableTypes()), op.propagatesInput());
+    }
+
+    @Override
+    public ILogicalOperator visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+        newInputList.addAll(op.getVariables());
+        return new DataSourceScanOperator(newInputList, op.getDataSource());
+    }
+
+    @Override
+    public ILogicalOperator visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new DistinctOperator(newExpressions);
+    }
+
+    @Override
+    public ILogicalOperator visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        return new ExchangeOperator();
+    }
+
+    @Override
+    public ILogicalOperator visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new WriteOperator(newExpressions, op.getDataSink());
+    }
+
+    @Override
+    public ILogicalOperator visitDistributeResultOperator(DistributeResultOperator op, Void arg)
+            throws AlgebricksException {
+        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newExpressions, op.getExpressions());
+        return new DistributeResultOperator(newExpressions, op.getDataSink());
+    }
+
+    @Override
+    public ILogicalOperator visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+        ArrayList<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newKeyExpressions, op.getKeyExpressions());
+        List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
+        WriteResultOperator writeResultOp = new WriteResultOperator(op.getDataSource(),
+                deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions);
+        writeResultOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
+        return writeResultOp;
+    }
+
+    @Override
+    public ILogicalOperator visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+        List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions());
+        List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
+        InsertDeleteOperator insertDeleteOp = new InsertDeleteOperator(op.getDataSource(),
+                deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions, op.getOperation(), op.isBulkload());
+        insertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
+        return insertDeleteOp;
+    }
+
+    @Override
+    public ILogicalOperator visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg)
+            throws AlgebricksException {
+        List<Mutable<ILogicalExpression>> newPrimaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
+        List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
+        Mutable<ILogicalExpression> newFilterExpression = new MutableObject<ILogicalExpression>(
+                ((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression());
+        List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newLSMComponentFilterExpressions, op.getAdditionalFilteringExpressions());
+        IndexInsertDeleteOperator indexInsertDeleteOp = new IndexInsertDeleteOperator(op.getDataSourceIndex(),
+                newPrimaryKeyExpressions, newSecondaryKeyExpressions, newFilterExpression, op.getOperation(),
+                op.isBulkload());
+        indexInsertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
+        return indexInsertDeleteOp;
+    }
+
+    @Override
+    public ILogicalOperator visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
+        List<Mutable<ILogicalExpression>> newPrimaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
+        List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
+        List<LogicalVariable> newTokenizeVars = new ArrayList<LogicalVariable>();
+        deepCopyVars(newTokenizeVars, op.getTokenizeVars());
+        Mutable<ILogicalExpression> newFilterExpression = new MutableObject<ILogicalExpression>(
+                ((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression());
+        List<Object> newTokenizeVarTypes = new ArrayList<Object>();
+        deepCopyObjects(newTokenizeVarTypes, op.getTokenizeVarTypes());
+
+        TokenizeOperator tokenizeOp = new TokenizeOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
+                newSecondaryKeyExpressions, newTokenizeVars, newFilterExpression, op.getOperation(), op.isBulkload(),
+                op.isPartitioned(), newTokenizeVarTypes);
+        return tokenizeOp;
+    }
+
+    @Override
+    public ILogicalOperator visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+        return new SinkOperator();
+    }
+
+    private void deepCopyExpressionRefs(List<Mutable<ILogicalExpression>> newExprs,
+            List<Mutable<ILogicalExpression>> oldExprs) {
+        for (Mutable<ILogicalExpression> oldExpr : oldExprs)
+            newExprs.add(new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue())
+                    .cloneExpression()));
+    }
+
+    private Mutable<ILogicalExpression> deepCopyExpressionRef(Mutable<ILogicalExpression> oldExpr) {
+        return new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression());
+    }
+
+    private List<LogicalVariable> deepCopyVars(List<LogicalVariable> newVars, List<LogicalVariable> oldVars) {
+        for (LogicalVariable oldVar : oldVars)
+            newVars.add(oldVar);
+        return newVars;
+    }
+
+    private List<Object> deepCopyObjects(List<Object> newObjs, List<Object> oldObjs) {
+        for (Object oldObj : oldObjs)
+            newObjs.add(oldObj);
+        return newObjs;
+    }
+
+    private List<Pair<IOrder, Mutable<ILogicalExpression>>> deepCopyOrderAndExpression(
+            List<Pair<IOrder, Mutable<ILogicalExpression>>> ordersAndExprs) {
+        List<Pair<IOrder, Mutable<ILogicalExpression>>> newOrdersAndExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+        for (Pair<IOrder, Mutable<ILogicalExpression>> pair : ordersAndExprs)
+            newOrdersAndExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(pair.first,
+                    deepCopyExpressionRef(pair.second)));
+        return newOrdersAndExprs;
+    }
+
+    @Override
+    public ILogicalOperator visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        return new ExtensionOperator(op.getNewInstanceOfDelegateOperator());
+    }
+
+    @Override
+    public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
+            throws AlgebricksException {
+        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+        newInputList.addAll(op.getVariables());
+        return new ExternalDataLookupOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()),
+                new ArrayList<Object>(op.getVariableTypes()), op.isPropagateInput(), op.getDataSource());
+    }
+
+    @Override
+    public ILogicalOperator visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
+        return new MaterializeOperator();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
new file mode 100644
index 0000000..1fa97a6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
@@ -0,0 +1,275 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ProducedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+    private Collection<LogicalVariable> producedVariables;
+
+    public ProducedVariableVisitor(Collection<LogicalVariable> producedVariables) throws AlgebricksException {
+        this.producedVariables = producedVariables;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
+            if (p.first != null) {
+                producedVariables.add(p.first);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
+            if (p.first != null) {
+                producedVariables.add(p.first);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+        List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op, usedVariables);
+        for (LogicalVariable v : op.getOutputVariables()) {
+            if (!usedVariables.contains(v)) {
+                producedVariables.add(v);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : op.getVariableMappings()) {
+            producedVariables.add(t.third);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getVariables());
+        LogicalVariable positionalVariable = op.getPositionalVariable();
+        if (positionalVariable != null) {
+            if (!producedVariables.contains(positionalVariable))
+                producedVariables.add(positionalVariable);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
+        producedVariables.addAll(op.getTokenizeVars());
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        op.getDelegate().getProducedVariables(producedVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
+        producedVariables.add(op.getVariables().get(0));
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
new file mode 100644
index 0000000..0b37b92
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
@@ -0,0 +1,330 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SchemaVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+    private Collection<LogicalVariable> schemaVariables;
+
+    public SchemaVariableVisitor(Collection<LogicalVariable> schemaVariables) {
+        this.schemaVariables = schemaVariables;
+    }
+
+    @Override
+    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+        schemaVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
+        List<LogicalVariable> allLiveVars = new ArrayList<LogicalVariable>();
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), allLiveVars);
+        }
+        VariableUtilities.getProducedVariables(op, allLiveVars);
+        /** put distinct vars first */
+        schemaVariables.addAll(op.getDistinctByVarList());
+        /** then other live vars */
+        for (LogicalVariable var : allLiveVars) {
+            if (!schemaVariables.contains(var)) {
+                schemaVariables.add(var);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
+            if (p.first != null) {
+                schemaVariables.add(p.first);
+            }
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
+            if (p.first != null) {
+                schemaVariables.add(p.first);
+            } else {
+                ILogicalExpression e = p.second.getValue();
+                if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                    schemaVariables.add(((VariableReferenceExpression) e).getVariableReference());
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
+        VariableUtilities.getLiveVariables(op.getSourceOperator(), schemaVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+        schemaVariables.addAll(op.getVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
+        // VariableUtilities.getProducedVariables(op, schemaVariables);
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+        schemaVariables.addAll(op.getOutputVariables());
+        return null;
+    }
+
+    @Override
+    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
+        }
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+        if (op.propagatesInput()) {
+            standardLayout(op);
+        } else {
+            VariableUtilities.getProducedVariables(op, schemaVariables);
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    private void standardLayout(ILogicalOperator op) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
+        }
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+    }
+
+    @Override
+    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
+        standardLayout(op);
+        return null;
+    }
+
+    @Override
+    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
+        ArrayList<LogicalVariable> liveVariables = new ArrayList<LogicalVariable>();
+        ArrayList<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+        //get used variables
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+        //live variables - used variables
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), liveVariables);
+        }
+        for (LogicalVariable v : liveVariables) {
+            if (!usedVariables.contains(v)) {
+                schemaVariables.add(v);
+            }
+        }
+        VariableUtilities.getProducedVariables(op, schemaVariables);
+        //+ produced variables
+        return null;
+    }
+
+}

[38/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
deleted file mode 100644
index d60d4f7..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class UnnestPOperator extends AbstractScanPOperator {
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.UNNEST;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        UnnestOperator unnest = (UnnestOperator) op;
-
-        int outCol = opSchema.findVariable(unnest.getVariable());
-        ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        boolean exit = false;
-        if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            exit = true;
-        } else {
-            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) unnestExpr;
-            if (fce.getKind() != FunctionKind.UNNEST) {
-                exit = true;
-            }
-        }
-        if (exit) {
-            throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call.");
-        }
-        UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr;
-        IUnnestingEvaluatorFactory unnestingFactory = expressionRuntimeProvider.createUnnestingFunctionFactory(agg,
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
-
-        // for position offset
-        ILogicalExpression posOffsetExpr = unnest.getPositionOffsetExpr();
-        IScalarEvaluatorFactory posOffsetExprEvalFactory = null;
-        if (posOffsetExpr != null) {
-            posOffsetExprEvalFactory = expressionRuntimeProvider.createEvaluatorFactory(posOffsetExpr,
-                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
-        }
-
-        int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
-        UnnestRuntimeFactory unnestRuntime = new UnnestRuntimeFactory(outCol, unnestingFactory, projectionList,
-                unnest.getPositionWriter(), posOffsetExprEvalFactory);
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        builder.contributeMicroOperator(unnest, unnestRuntime, recDesc);
-        ILogicalOperator src = unnest.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, unnest, 0);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
deleted file mode 100644
index f399184..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class WriteResultPOperator extends AbstractPhysicalOperator {
-
-    private LogicalVariable payload;
-    private List<LogicalVariable> keys;
-    private IDataSource<?> dataSource;
-    private final List<LogicalVariable> additionalFilteringKeys;
-
-    public WriteResultPOperator(IDataSource<?> dataSource, LogicalVariable payload, List<LogicalVariable> keys,
-            List<LogicalVariable> additionalFilteringKeys) {
-        this.dataSource = dataSource;
-        this.payload = payload;
-        this.keys = keys;
-        this.additionalFilteringKeys = additionalFilteringKeys;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + " " + payload + " " + keys;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.WRITE_RESULT;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
-        scanVariables.addAll(keys);
-        scanVariables.add(new LogicalVariable(-1));
-        IPhysicalPropertiesVector r = dataSource.getPropertiesProvider().computePropertiesVector(scanVariables);
-        IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
-        requirements[0] = r;
-        return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @SuppressWarnings({ "rawtypes", "unchecked" })
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        WriteResultOperator writeResultOp = (WriteResultOperator) op;
-        IMetadataProvider mp = context.getMetadataProvider();
-
-        JobSpecification spec = builder.getJobSpec();
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getWriteResultRuntime(
-                dataSource, propagatedSchema, keys, payload, additionalFilteringKeys, context, spec);
-
-        builder.contributeHyracksOperator(writeResultOp, runtimeAndConstraints.first);
-        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = writeResultOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, writeResultOp, 0);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
deleted file mode 100644
index 8048bf1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.plan;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-
-/*
- * Author: Guangqiang Li
- * Created on Jul 9, 2009 
- */
-public class ALogicalPlanImpl implements ILogicalPlan {
-    private List<Mutable<ILogicalOperator>> roots;
-
-    public ALogicalPlanImpl() {
-        this.roots = new ArrayList<Mutable<ILogicalOperator>>();
-    }
-
-    public ALogicalPlanImpl(List<Mutable<ILogicalOperator>> roots) {
-        this.roots = roots;
-    }
-
-    public ALogicalPlanImpl(Mutable<ILogicalOperator> root) {
-        roots = new ArrayList<Mutable<ILogicalOperator>>(1);
-        roots.add(root);
-    }
-
-    public List<Mutable<ILogicalOperator>> getRoots() {
-        return roots;
-    }
-
-    public void setRoots(List<Mutable<ILogicalOperator>> roots) {
-        this.roots = roots;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
deleted file mode 100644
index 5f77352..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2014 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-
-public class LogicalExpressionPrettyPrintVisitor implements ILogicalExpressionVisitor<String, Integer> {
-
-	@Override
-	public String visitConstantExpression(ConstantExpression expr, Integer indent)
-			throws AlgebricksException {
-		return expr.toString();
-	}
-
-	@Override
-	public String visitVariableReferenceExpression(
-			VariableReferenceExpression expr, Integer indent)
-			throws AlgebricksException {
-		return expr.toString();
-	}
-
-	@Override
-	public String visitAggregateFunctionCallExpression(
-			AggregateFunctionCallExpression expr, Integer indent)
-			throws AlgebricksException {
-		return expr.toString();
-	}
-
-	@Override
-	public String visitScalarFunctionCallExpression(
-			ScalarFunctionCallExpression expr, Integer indent)
-			throws AlgebricksException {
-		return expr.toString();
-	}
-
-	@Override
-	public String visitStatefulFunctionCallExpression(
-			StatefulFunctionCallExpression expr, Integer indent)
-			throws AlgebricksException {
-		return expr.toString();
-	}
-
-	@Override
-	public String visitUnnestingFunctionCallExpression(
-			UnnestingFunctionCallExpression expr, Integer indent)
-			throws AlgebricksException {
-		return expr.toString();
-	}
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
deleted file mode 100644
index 2c924d5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class LogicalOperatorPrettyPrintVisitor implements ILogicalOperatorVisitor<String, Integer> {
-
-    ILogicalExpressionVisitor<String, Integer> exprVisitor;
-
-    public LogicalOperatorPrettyPrintVisitor() {
-        exprVisitor = new LogicalExpressionPrettyPrintVisitor();
-    }
-
-    public LogicalOperatorPrettyPrintVisitor(ILogicalExpressionVisitor<String, Integer> exprVisitor) {
-        this.exprVisitor = exprVisitor;
-    }
-
-    @Override
-    public String visitAggregateOperator(AggregateOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("aggregate ").append(op.getVariables()).append(" <- ");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitRunningAggregateOperator(RunningAggregateOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("running-aggregate ").append(op.getVariables()).append(" <- ");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("empty-tuple-source");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("group by (");
-        pprintVeList(buffer, op.getGroupByList(), indent);
-        buffer.append(") decor (");
-        pprintVeList(buffer, op.getDecorList(), indent);
-        buffer.append(") {");
-        printNestedPlans(op, indent, buffer);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitDistinctOperator(DistinctOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("distinct " + "(");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        buffer.append(")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitInnerJoinOperator(InnerJoinOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("join (").append(op.getCondition().getValue().accept(exprVisitor, indent))
-                .append(")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("left outer join (")
-                .append(op.getCondition().getValue().accept(exprVisitor, indent)).append(")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("nested tuple source");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitOrderOperator(OrderOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("order ");
-        for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
-            String fst;
-            switch (p.first.getKind()) {
-                case ASC: {
-                    fst = "ASC";
-                    break;
-                }
-                case DESC: {
-                    fst = "DESC";
-                    break;
-                }
-                default: {
-                    fst = p.first.getExpressionRef().toString();
-                }
-            }
-            buffer.append("(" + fst + ", " + p.second.getValue().accept(exprVisitor, indent) + ") ");
-        }
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitAssignOperator(AssignOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("assign ").append(op.getVariables()).append(" <- ");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitWriteOperator(WriteOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("write ");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitDistributeResultOperator(DistributeResultOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("distribute result ");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitWriteResultOperator(WriteResultOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("load ").append(op.getDataSource()).append(" from ")
-                .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)).append(" partitioned by ");
-        pprintExprList(op.getKeyExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitSelectOperator(SelectOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("select (").append(op.getCondition().getValue().accept(exprVisitor, indent))
-                .append(")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitProjectOperator(ProjectOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("project " + "(" + op.getVariables() + ")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitPartitioningSplitOperator(PartitioningSplitOperator op, Integer indent)
-            throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("partitioning-split (");
-        pprintExprList(op.getExpressions(), buffer, indent);
-        buffer.append(")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitSubplanOperator(SubplanOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("subplan {");
-        printNestedPlans(op, indent, buffer);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitUnionOperator(UnionAllOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("union");
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> v : op.getVariableMappings()) {
-            buffer.append(" (" + v.first + ", " + v.second + ", " + v.third + ")");
-        }
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitUnnestOperator(UnnestOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("unnest " + op.getVariable());
-        if (op.getPositionalVariable() != null) {
-            buffer.append(" at " + op.getPositionalVariable());
-        }
-        buffer.append(" <- " + op.getExpressionRef().getValue().accept(exprVisitor, indent));
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append(
-                "unnest-map " + op.getVariables() + " <- "
-                        + op.getExpressionRef().getValue().accept(exprVisitor, indent));
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitDataScanOperator(DataSourceScanOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append(
-                "data-scan " + op.getProjectVariables() + "<-" + op.getVariables() + " <- " + op.getDataSource());
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitLimitOperator(LimitOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("limit " + op.getMaxObjects().getValue().accept(exprVisitor, indent));
-        ILogicalExpression offset = op.getOffset().getValue();
-        if (offset != null) {
-            buffer.append(", " + offset.accept(exprVisitor, indent));
-        }
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitExchangeOperator(ExchangeOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("exchange ");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitScriptOperator(ScriptOperator op, Integer indent) {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append(
-                "script (in: " + op.getInputVariables() + ") (out: " + op.getOutputVariables() + ")");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitReplicateOperator(ReplicateOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("replicate ");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitMaterializeOperator(MaterializeOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("materialize ");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitInsertDeleteOperator(InsertDeleteOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
-        addIndent(buffer, indent).append(header).append(op.getDataSource()).append(" from ")
-                .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)).append(" partitioned by ");
-        pprintExprList(op.getPrimaryKeyExpressions(), buffer, indent);
-        if (op.isBulkload()) {
-            buffer.append(" [bulkload]");
-        }
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Integer indent)
-            throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
-        addIndent(buffer, indent).append(header).append(op.getIndexName()).append(" on ")
-                .append(op.getDataSourceIndex().getDataSource()).append(" from ");
-        pprintExprList(op.getSecondaryKeyExpressions(), buffer, indent);
-        if (op.isBulkload()) {
-            buffer.append(" [bulkload]");
-        }
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitTokenizeOperator(TokenizeOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("tokenize ").append(op.getTokenizeVars()).append(" <- ");
-        pprintExprList(op.getSecondaryKeyExpressions(), buffer, indent);
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitSinkOperator(SinkOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("sink");
-        return buffer.toString();
-    }
-
-    @Override
-    public String visitExtensionOperator(ExtensionOperator op, Integer indent) throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append(op.toString());
-        return buffer.toString();
-    }
-
-    protected static final StringBuilder addIndent(StringBuilder buffer, int level) {
-        for (int i = 0; i < level; ++i) {
-            buffer.append(' ');
-        }
-        return buffer;
-    }
-
-    protected void printNestedPlans(AbstractOperatorWithNestedPlans op, Integer indent, StringBuilder buffer)
-            throws AlgebricksException {
-        boolean first = true;
-        if (op.getNestedPlans().isEmpty()) {
-            buffer.append("}");
-        } else {
-            for (ILogicalPlan p : op.getNestedPlans()) {
-                // PrettyPrintUtil.indent(buffer, level + 10).append("var " +
-                // p.first + ":\n");
-                buffer.append("\n");
-                if (first) {
-                    first = false;
-                } else {
-                    addIndent(buffer, indent).append("       {\n");
-                }
-                PlanPrettyPrinter.printPlan(p, buffer, this, indent + 10);
-                addIndent(buffer, indent).append("       }");
-            }
-        }
-    }
-
-    protected void pprintExprList(List<Mutable<ILogicalExpression>> expressions, StringBuilder buffer, Integer indent)
-            throws AlgebricksException {
-        buffer.append("[");
-        boolean first = true;
-        for (Mutable<ILogicalExpression> exprRef : expressions) {
-            if (first) {
-                first = false;
-            } else {
-                buffer.append(", ");
-            }
-            buffer.append(exprRef.getValue().accept(exprVisitor, indent));
-        }
-        buffer.append("]");
-    }
-
-    protected void pprintVeList(StringBuilder sb, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList,
-            Integer indent) throws AlgebricksException {
-        sb.append("[");
-        boolean fst = true;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
-            if (fst) {
-                fst = false;
-            } else {
-                sb.append("; ");
-            }
-            if (ve.first != null) {
-                sb.append(ve.first + " := " + ve.second);
-            } else {
-                sb.append(ve.second.getValue().accept(exprVisitor, indent));
-            }
-        }
-        sb.append("]");
-    }
-
-    @Override
-    public String visitExternalDataLookupOperator(ExternalDataLookupOperator op, Integer indent)
-            throws AlgebricksException {
-        StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append(
-                "external-instant-lookup " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
-        return buffer.toString();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java
deleted file mode 100644
index 85274de..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPlotter.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Random;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-
-public class PlanPlotter {
-
-    static Random randomGenerator = new Random();
-
-    public static void printLogicalPlan(ILogicalPlan plan) throws AlgebricksException {
-        int indent = 5;
-        StringBuilder out = new StringBuilder();
-        int randomInt = 10000 + randomGenerator.nextInt(100);
-        appendln(out, "digraph G {");
-        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
-            printVisualizationGraph((AbstractLogicalOperator) root.getValue(), indent, out, "", randomInt);
-        }
-        appendln(out, "\n}\n}");
-        try {
-            File file = File.createTempFile("logicalPlan", ".txt");
-            FileUtils.writeStringToFile(file, out.toString());
-            file.deleteOnExit();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    public static void printOptimizedLogicalPlan(ILogicalPlan plan) throws AlgebricksException {
-        int indent = 5;
-        StringBuilder out = new StringBuilder();
-        int randomInt = 10000 + randomGenerator.nextInt(100);
-        appendln(out, "digraph G {");
-        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
-            printVisualizationGraph((AbstractLogicalOperator) root.getValue(), indent, out, "", randomInt);
-        }
-        appendln(out, "\n}\n}");
-        try {
-            File file = File.createTempFile("logicalOptimizedPlan", ".txt");
-            FileUtils.writeStringToFile(file, out.toString());
-            file.deleteOnExit();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    /*
-     * DFS traversal function. Calls iteratively all children, and for each calls itself recursively
-     * Includes slim-maps only (no gathering of mappers to one)
-     */
-    public static void printVisualizationGraph(AbstractLogicalOperator op, int indent, StringBuilder out,
-            String current_supernode_name, int randomInt) {
-        if (!op.getInputs().isEmpty()) {
-            //String stringToVisualize = op.toStringForVisualizationGraph();
-            String stringToVisualize = op.getOperatorTag().name();
-            int firstOccurenceOf_ = stringToVisualize.indexOf("_");
-            String supernode_current = stringToVisualize.substring(firstOccurenceOf_ + 1, stringToVisualize.length());
-            if (current_supernode_name.isEmpty()) {
-                current_supernode_name = supernode_current;
-                appendln(out, new String("subgraph cluster_" + supernode_current + " {"));
-                pad(out, indent);
-                appendln(out, new String("node [style=filled, color = lightgray];"));
-                pad(out, indent);
-                appendln(out, new String("color=blue;"));
-                pad(out, indent);
-                String op_id = op.toString().substring(op.toString().indexOf("@") + 1, op.toString().length());
-                appendln(out, new String("label = \"" + supernode_current + "ID" + op_id + "\";"));
-                pad(out, indent);
-            }
-
-            for (Mutable<ILogicalOperator> i : op.getInputs()) {
-                String op_id = i.toString().substring(i.toString().indexOf("@") + 1, i.toString().length());
-                String logOpStr = ((AbstractLogicalOperator) i.getValue()).getOperatorTag().name() + "ID" + op_id;
-                firstOccurenceOf_ = logOpStr.indexOf("_");
-                String supernode_child = logOpStr.substring(firstOccurenceOf_ + 1, logOpStr.length());
-                if (!supernode_current.equals(supernode_child)) {
-                    appendln(out, new String("node [style=filled, color = lightgray];"));
-                    pad(out, indent);
-                    appendln(out, new String("color=blue"));
-                    pad(out, indent);
-                    appendln(out, new String("label = \"" + supernode_child + "\";"));
-                    pad(out, indent);
-                }
-
-                op_id = op.toString().substring(op.toString().indexOf("@") + 1, op.toString().length());
-                appendln(out, stringToVisualize + "ID" + op_id + "[style = filled]");
-                AbstractLogicalOperator child = (AbstractLogicalOperator) i.getValue();
-
-                pad(out, indent);
-                String op_id1 = op.toString().substring(op.toString().indexOf("@") + 1, op.toString().length());
-                append(out, op.getOperatorTag().name() + "ID" + op_id1 + " -> ");
-                String opc_id = child.toString()
-                        .substring(child.toString().indexOf("@") + 1, child.toString().length());
-                appendln(out, child.getOperatorTag().name() + "ID" + opc_id);
-
-                printVisualizationGraph(child, indent, out, supernode_current, (randomGenerator.nextInt(100) + 10000));
-
-            }
-        }
-
-    }
-
-    private static void appendln(StringBuilder buf, String s) {
-        buf.append(s);
-        buf.append("\n");
-    }
-
-    private static void append(StringBuilder buf, String s) {
-        buf.append(s);
-    }
-
-    private static void pad(StringBuilder buf, int indent) {
-        for (int i = 0; i < indent; ++i) {
-            buf.append(' ');
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
deleted file mode 100644
index 33c106b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-
-public class PlanPrettyPrinter {
-    public static void printPlan(ILogicalPlan plan, StringBuilder out, LogicalOperatorPrettyPrintVisitor pvisitor,
-            int indent) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
-            printOperator((AbstractLogicalOperator) root.getValue(), out, pvisitor, indent);
-        }
-    }
-
-    public static void printPhysicalOps(ILogicalPlan plan, StringBuilder out, int indent) {
-        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
-            printPhysicalOperator((AbstractLogicalOperator) root.getValue(), indent, out);
-        }
-    }
-
-    public static void printOperator(AbstractLogicalOperator op, StringBuilder out,
-            LogicalOperatorPrettyPrintVisitor pvisitor, int indent) throws AlgebricksException {
-        out.append(op.accept(pvisitor, indent));
-        IPhysicalOperator pOp = op.getPhysicalOperator();
-
-        if (pOp != null) {
-            out.append("\n");
-            pad(out, indent);
-            appendln(out, "-- " + pOp.toString() + "  |" + op.getExecutionMode() + "|");
-        } else {
-            appendln(out, " -- |" + op.getExecutionMode() + "|");
-        }
-
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            printOperator((AbstractLogicalOperator) i.getValue(), out, pvisitor, indent + 2);
-        }
-
-    }
-
-    public static void printPhysicalOperator(AbstractLogicalOperator op, int indent, StringBuilder out) {
-        IPhysicalOperator pOp = op.getPhysicalOperator();
-        pad(out, indent);
-        appendln(out, "-- " + pOp.toString() + "  |" + op.getExecutionMode() + "|");
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans opNest = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : opNest.getNestedPlans()) {
-                pad(out, indent + 8);
-                appendln(out, "{");
-                printPhysicalOps(p, out, indent + 10);
-                pad(out, indent + 8);
-                appendln(out, "}");
-            }
-        }
-
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            printPhysicalOperator((AbstractLogicalOperator) i.getValue(), indent + 2, out);
-        }
-
-    }
-
-    private static void appendln(StringBuilder buf, String s) {
-        buf.append(s);
-        buf.append("\n");
-    }
-
-    private static void pad(StringBuilder buf, int indent) {
-        for (int i = 0; i < indent; ++i) {
-            buf.append(' ');
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
deleted file mode 100644
index b757bbb..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public abstract class AbstractGroupingProperty {
-    protected Set<LogicalVariable> columnSet;
-
-    public AbstractGroupingProperty(Set<LogicalVariable> columnSet) {
-        this.columnSet = columnSet;
-    }
-
-    public Set<LogicalVariable> getColumnSet() {
-        return columnSet;
-    }
-
-    public final void normalizeGroupingColumns(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
-            List<FunctionalDependency> fds) {
-        replaceGroupingColumnsByEqClasses(equivalenceClasses);
-        applyFDsToGroupingColumns(fds);
-    }
-
-    private void replaceGroupingColumnsByEqClasses(Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
-        if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
-            return;
-        }
-        Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
-        for (LogicalVariable v : columnSet) {
-            EquivalenceClass ec = equivalenceClasses.get(v);
-            if (ec == null) {
-                norm.add(v);
-            } else {
-                if (ec.representativeIsConst()) {
-                    // trivially satisfied, so the var. can be removed
-                } else {
-                    norm.add(ec.getVariableRepresentative());
-                }
-            }
-        }
-        columnSet = norm;
-    }
-
-    private void applyFDsToGroupingColumns(List<FunctionalDependency> fds) {
-        // the set of vars. is unordered
-        // so we try all FDs on all variables (incomplete algo?)
-        if (fds == null || fds.isEmpty()) {
-            return;
-        }
-        Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
-        for (LogicalVariable v : columnSet) {
-            boolean isImpliedByAnFD = false;
-            for (FunctionalDependency fdep : fds) {
-                if (columnSet.containsAll(fdep.getHead()) && fdep.getTail().contains(v)) {
-                    isImpliedByAnFD = true;
-                    norm.addAll(fdep.getHead());
-                    break;
-                }
-
-            }
-            if (!isImpliedByAnFD) {
-                norm.add(v);
-            }
-        }
-        columnSet = norm;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
deleted file mode 100644
index 53aea68..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public class BroadcastPartitioningProperty implements IPartitioningProperty {
-
-    private INodeDomain domain;
-
-    public BroadcastPartitioningProperty(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-    @Override
-    public PartitioningType getPartitioningType() {
-        return PartitioningType.BROADCAST;
-    }
-
-    @Override
-    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
-        // do nothing
-    }
-
-    @Override
-    public void getColumns(Collection<LogicalVariable> columns) {
-    }
-
-    @Override
-    public INodeDomain getNodeDomain() {
-        return domain;
-    }
-
-    @Override
-    public void setNodeDomain(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
deleted file mode 100644
index 226f02c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-public class DefaultNodeGroupDomain implements INodeDomain {
-
-    private String groupName;
-
-    public DefaultNodeGroupDomain(String groupName) {
-        this.groupName = groupName;
-    }
-
-    @Override
-    public boolean sameAs(INodeDomain domain) {
-        return true;
-    }
-
-    @Override
-    public String toString() {
-        return "AsterixDomain(" + groupName + ")";
-    }
-
-    @Override
-    public Integer cardinality() {
-        return null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
deleted file mode 100644
index 5fbcdb9..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-
-public class FileSplitDomain implements INodeDomain {
-
-    private FileSplit[] splits;
-
-    public FileSplitDomain(FileSplit[] splits) {
-        this.splits = splits;
-    }
-
-    @Override
-    public Integer cardinality() {
-        return splits.length;
-    }
-
-    @Override
-    public String toString() {
-        StringBuilder sb = new StringBuilder();
-        sb.append("FileSplitDomain[");
-        boolean fst = true;
-        for (FileSplit fs : splits) {
-            if (fst) {
-                fst = false;
-            } else {
-                sb.append(", ");
-            }
-            sb.append(fs.getNodeName() + ":" + fs.getLocalFile());
-        }
-        sb.append(']');
-        return sb.toString();
-    }
-
-    @Override
-    public boolean sameAs(INodeDomain domain) {
-        if (!(domain instanceof FileSplitDomain)) {
-            return false;
-        }
-        FileSplitDomain fsd = (FileSplitDomain) domain;
-        if (fsd.splits.length != splits.length) {
-            return false;
-        }
-        // conservative approach...
-        for (int i = 0; i < splits.length; i++) {
-            if (!ncEq(splits[i], fsd.splits[i])) {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
-    private boolean ncEq(FileSplit fs1, FileSplit fs2) {
-        return fs1.getNodeName().equals(fs2.getNodeName());
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
deleted file mode 100644
index 822cdc3..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-
-public class FilteredVariablePropagationPolicy extends VariablePropagationPolicy {
-
-    List<LogicalVariable> toPropagate;
-
-    public FilteredVariablePropagationPolicy(List<LogicalVariable> varList) {
-        toPropagate = varList;
-    }
-
-    @Override
-    public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
-        for (LogicalVariable v : toPropagate) {
-            target.addVariable(v);
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
deleted file mode 100644
index 5bf4637..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public final class FunctionalDependency {
-    private List<LogicalVariable> head;
-    private List<LogicalVariable> tail;
-
-    public FunctionalDependency(List<LogicalVariable> head, List<LogicalVariable> tail) {
-        this.head = head;
-        this.tail = tail;
-    }
-
-    public List<LogicalVariable> getHead() {
-        return head;
-    }
-
-    public List<LogicalVariable> getTail() {
-        return tail;
-    }
-
-    @Override
-    public String toString() {
-        return head + "->" + tail;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof FunctionalDependency)) {
-            return false;
-        } else {
-            FunctionalDependency fd = (FunctionalDependency) obj;
-            return fd.getHead().equals(this.head) && fd.getTail().equals(this.tail);
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
deleted file mode 100644
index c32fd67..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface ILocalStructuralProperty extends IStructuralProperty {
-    public enum PropertyType {
-        LOCAL_GROUPING_PROPERTY,
-        LOCAL_ORDER_PROPERTY
-    }
-
-    public void getVariables(Collection<LogicalVariable> variables);
-
-    public PropertyType getPropertyType();
-
-    /**
-     * Returns the retained property regarding to a collection of variables,
-     * e.g., some variables used in the property may not exist in the input
-     * collection and hence the data property changes.
-     * 
-     * @param vars
-     *            , an input collection of variables
-     * @return the retained data property.
-     */
-    public ILocalStructuralProperty retainVariables(Collection<LogicalVariable> vars);
-
-    /**
-     * Returns the additional data property within each group, which is dictated by the group keys.
-     * 
-     * @param vars
-     *            , group keys.
-     * @return the additional data property within each group.
-     */
-    public ILocalStructuralProperty regardToGroup(Collection<LogicalVariable> groupKeys);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
deleted file mode 100644
index 175f041..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-public interface ILogicalPropertiesVector {
-    public Integer getNumberOfTuples();
-
-    public Integer getMaxOutputFrames();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java
deleted file mode 100644
index 17e97ec..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-public interface INodeDomain {
-    public boolean sameAs(INodeDomain domain);
-
-    /** @return the number of nodes in that domain or null if unknown */
-    public Integer cardinality();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
deleted file mode 100644
index bb8cfe2..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IPartitioningProperty extends IStructuralProperty {
-    public enum PartitioningType {
-        UNPARTITIONED, RANDOM, BROADCAST, UNORDERED_PARTITIONED, ORDERED_PARTITIONED
-    }
-
-    static final INodeDomain DOMAIN_FOR_UNPARTITIONED_DATA = new INodeDomain() {
-        @Override
-        public boolean sameAs(INodeDomain domain) {
-            return domain == this;
-        }
-
-        @Override
-        public Integer cardinality() {
-            return null;
-        }
-    };
-
-    public static final IPartitioningProperty UNPARTITIONED = new IPartitioningProperty() {
-
-        @Override
-        public PartitioningType getPartitioningType() {
-            return PartitioningType.UNPARTITIONED;
-        }
-
-        @Override
-        public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
-            // do nothing
-        }
-
-        @Override
-        public void getColumns(Collection<LogicalVariable> columns) {
-        }
-
-        @Override
-        public INodeDomain getNodeDomain() {
-            return DOMAIN_FOR_UNPARTITIONED_DATA;
-        }
-
-        @Override
-        public String toString() {
-            return getPartitioningType().toString();
-        }
-
-        @Override
-        public void setNodeDomain(INodeDomain domain) {
-            throw new IllegalStateException();
-        }
-    };
-
-    public abstract PartitioningType getPartitioningType();
-
-    public abstract void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
-            List<FunctionalDependency> fds);
-
-    public abstract INodeDomain getNodeDomain();
-
-    public abstract void setNodeDomain(INodeDomain domain);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
deleted file mode 100644
index cd9eb96..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-/**
- * Implements constraints in between requirements for the children of the same
- * operator.
- */
-
-public interface IPartitioningRequirementsCoordinator {
-
-    public static IPartitioningRequirementsCoordinator NO_COORDINATION = new IPartitioningRequirementsCoordinator() {
-
-        @Override
-        public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements,
-                IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context) {
-            return new Pair<Boolean, IPartitioningProperty>(true, requirements);
-        }
-    };
-
-    public static IPartitioningRequirementsCoordinator EQCLASS_PARTITIONING_COORDINATOR = new IPartitioningRequirementsCoordinator() {
-
-        @Override
-        public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty rqdpp,
-                IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context)
-                throws AlgebricksException {
-            if (firstDeliveredPartitioning != null
-                    && firstDeliveredPartitioning.getPartitioningType() == rqdpp.getPartitioningType()) {
-                switch (rqdpp.getPartitioningType()) {
-                    case UNORDERED_PARTITIONED: {
-                        UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
-                        Set<LogicalVariable> set1 = upp1.getColumnSet();
-                        UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) rqdpp;
-                        Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
-                        Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
-                        Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
-
-                        // coordinate from an existing partition property
-                        // (firstDeliveredPartitioning)
-                        for (LogicalVariable v : set1) {
-                            EquivalenceClass ecFirst = eqmap.get(v);
-                            for (LogicalVariable r : uppreq.getColumnSet()) {
-                                EquivalenceClass ec = eqmap.get(r);
-                                if (ecFirst == ec) {
-                                    covered.add(v);
-                                    modifuppreq.add(r);
-                                    break;
-                                }
-                            }
-                        }
-
-                        if (!covered.equals(set1)) {
-                            throw new AlgebricksException("Could not modify " + rqdpp
-                                    + " to agree with partitioning property " + firstDeliveredPartitioning
-                                    + " delivered by previous input operator.");
-                        }
-                        UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq,
-                                rqdpp.getNodeDomain());
-                        return new Pair<Boolean, IPartitioningProperty>(false, upp2);
-                    }
-                    case ORDERED_PARTITIONED: {
-                        throw new NotImplementedException();
-                    }
-                }
-            }
-            return new Pair<Boolean, IPartitioningProperty>(true, rqdpp);
-        }
-
-    };
-
-    public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements,
-            IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
deleted file mode 100644
index 45bf1ec..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IPhysicalPropertiesVector {
-
-    public IPartitioningProperty getPartitioningProperty();
-
-    public List<ILocalStructuralProperty> getLocalProperties();
-
-    /**
-     * 
-     * @param reqd
-     *            vector of required properties
-     * @param equivalenceClasses
-     * @param fds
-     * @return a vector of properties from pvector that are not delivered by the
-     *         current vector or null if none
-     */
-    public IPhysicalPropertiesVector getUnsatisfiedPropertiesFrom(IPhysicalPropertiesVector reqd,
-            boolean mayExpandProperties, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
-            List<FunctionalDependency> fds);
-
-    public IPhysicalPropertiesVector clone();
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
deleted file mode 100644
index 716b459..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-
-public interface IPropertiesComputer {
-
-    // can return null
-    public IPartitioningProperty computePartitioningProperty(ILogicalExpression expr);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
deleted file mode 100644
index bf8aaa5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IStructuralProperty {
-    public void getColumns(Collection<LogicalVariable> columns);
-}

[14/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
deleted file mode 100644
index 07621ce..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
+++ /dev/null
@@ -1,502 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule {
-
-    private final HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> childrenToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
-    private final List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
-    private final List<List<Mutable<ILogicalOperator>>> equivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
-    private final HashMap<Mutable<ILogicalOperator>, BitSet> opToCandidateInputs = new HashMap<Mutable<ILogicalOperator>, BitSet>();
-    private final HashMap<Mutable<ILogicalOperator>, MutableInt> clusterMap = new HashMap<Mutable<ILogicalOperator>, MutableInt>();
-    private final HashMap<Integer, BitSet> clusterWaitForMap = new HashMap<Integer, BitSet>();
-    private int lastUsedClusterId = 0;
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT
-                && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) {
-            return false;
-        }
-        if (!roots.contains(op))
-            roots.add(new MutableObject<ILogicalOperator>(op));
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT
-                && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) {
-            return false;
-        }
-        boolean rewritten = false;
-        boolean changed = false;
-        if (roots.size() > 0) {
-            do {
-                changed = false;
-                // applying the rewriting until fixpoint
-                topDownMaterialization(roots);
-                genCandidates(context);
-                removeTrivialShare();
-                if (equivalenceClasses.size() > 0)
-                    changed = rewrite(context);
-                if (!rewritten)
-                    rewritten = changed;
-                equivalenceClasses.clear();
-                childrenToParents.clear();
-                opToCandidateInputs.clear();
-                clusterMap.clear();
-                clusterWaitForMap.clear();
-                lastUsedClusterId = 0;
-            } while (changed);
-            roots.clear();
-        }
-        return rewritten;
-    }
-
-    private void removeTrivialShare() {
-        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
-            for (int i = candidates.size() - 1; i >= 0; i--) {
-                Mutable<ILogicalOperator> opRef = candidates.get(i);
-                AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
-                if (aop.getOperatorTag() == LogicalOperatorTag.EXCHANGE)
-                    aop = (AbstractLogicalOperator) aop.getInputs().get(0).getValue();
-                if (aop.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE)
-                    candidates.remove(i);
-            }
-        }
-        for (int i = equivalenceClasses.size() - 1; i >= 0; i--)
-            if (equivalenceClasses.get(i).size() < 2)
-                equivalenceClasses.remove(i);
-    }
-
-    private boolean rewrite(IOptimizationContext context) throws AlgebricksException {
-        boolean changed = false;
-        for (List<Mutable<ILogicalOperator>> members : equivalenceClasses) {
-            if (rewriteForOneEquivalentClass(members, context))
-                changed = true;
-        }
-        return changed;
-    }
-
-    private boolean rewriteForOneEquivalentClass(List<Mutable<ILogicalOperator>> members, IOptimizationContext context)
-            throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> group = new ArrayList<Mutable<ILogicalOperator>>();
-        boolean rewritten = false;
-        while (members.size() > 0) {
-            group.clear();
-            Mutable<ILogicalOperator> candidate = members.remove(members.size() - 1);
-            group.add(candidate);
-            for (int i = members.size() - 1; i >= 0; i--) {
-                Mutable<ILogicalOperator> peer = members.get(i);
-                if (IsomorphismUtilities.isOperatorIsomorphic(candidate.getValue(), peer.getValue())) {
-                    group.add(peer);
-                    members.remove(i);
-                }
-            }
-            boolean[] materializationFlags = computeMaterilizationFlags(group);
-            if (group.isEmpty()) {
-                continue;
-            }
-            candidate = group.get(0);
-            ReplicateOperator rop = new ReplicateOperator(group.size(), materializationFlags);
-            rop.setPhysicalOperator(new ReplicatePOperator());
-            rop.setExecutionMode(ExecutionMode.PARTITIONED);
-            Mutable<ILogicalOperator> ropRef = new MutableObject<ILogicalOperator>(rop);
-            AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getValue();
-            List<Mutable<ILogicalOperator>> originalCandidateParents = childrenToParents.get(candidate);
-
-            if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-                rop.getInputs().add(candidate);
-            } else {
-                AbstractLogicalOperator beforeExchange = new ExchangeOperator();
-                beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator());
-                Mutable<ILogicalOperator> beforeExchangeRef = new MutableObject<ILogicalOperator>(beforeExchange);
-                beforeExchange.getInputs().add(candidate);
-                context.computeAndSetTypeEnvironmentForOperator(beforeExchange);
-                rop.getInputs().add(beforeExchangeRef);
-            }
-            context.computeAndSetTypeEnvironmentForOperator(rop);
-
-            for (Mutable<ILogicalOperator> parentRef : originalCandidateParents) {
-                AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getValue();
-                int index = parent.getInputs().indexOf(candidate);
-                if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-                    parent.getInputs().set(index, ropRef);
-                    rop.getOutputs().add(parentRef);
-                } else {
-                    AbstractLogicalOperator exchange = new ExchangeOperator();
-                    exchange.setPhysicalOperator(new OneToOneExchangePOperator());
-                    MutableObject<ILogicalOperator> exchangeRef = new MutableObject<ILogicalOperator>(exchange);
-                    exchange.getInputs().add(ropRef);
-                    rop.getOutputs().add(exchangeRef);
-                    context.computeAndSetTypeEnvironmentForOperator(exchange);
-                    parent.getInputs().set(index, exchangeRef);
-                    context.computeAndSetTypeEnvironmentForOperator(parent);
-                }
-            }
-            List<LogicalVariable> liveVarsNew = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(candidate.getValue(), liveVarsNew);
-            ArrayList<Mutable<ILogicalExpression>> assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
-            for (LogicalVariable liveVar : liveVarsNew)
-                assignExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(liveVar)));
-            for (Mutable<ILogicalOperator> ref : group) {
-                if (ref.equals(candidate))
-                    continue;
-                ArrayList<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
-                Map<LogicalVariable, LogicalVariable> variableMappingBack = new HashMap<LogicalVariable, LogicalVariable>();
-                IsomorphismUtilities.mapVariablesTopDown(ref.getValue(), candidate.getValue(), variableMappingBack);
-                for (int i = 0; i < liveVarsNew.size(); i++) {
-                    liveVars.add(variableMappingBack.get(liveVarsNew.get(i)));
-                }
-
-                AbstractLogicalOperator assignOperator = new AssignOperator(liveVars, assignExprs);
-                assignOperator.setPhysicalOperator(new AssignPOperator());
-                AbstractLogicalOperator projectOperator = new ProjectOperator(liveVars);
-                projectOperator.setPhysicalOperator(new StreamProjectPOperator());
-                AbstractLogicalOperator exchOp = new ExchangeOperator();
-                exchOp.setPhysicalOperator(new OneToOneExchangePOperator());
-                exchOp.getInputs().add(ropRef);
-                MutableObject<ILogicalOperator> exchOpRef = new MutableObject<ILogicalOperator>(exchOp);
-                rop.getOutputs().add(exchOpRef);
-                assignOperator.getInputs().add(exchOpRef);
-                projectOperator.getInputs().add(new MutableObject<ILogicalOperator>(assignOperator));
-
-                // set the types
-                context.computeAndSetTypeEnvironmentForOperator(exchOp);
-                context.computeAndSetTypeEnvironmentForOperator(assignOperator);
-                context.computeAndSetTypeEnvironmentForOperator(projectOperator);
-
-                List<Mutable<ILogicalOperator>> parentOpList = childrenToParents.get(ref);
-                for (Mutable<ILogicalOperator> parentOpRef : parentOpList) {
-                    AbstractLogicalOperator parentOp = (AbstractLogicalOperator) parentOpRef.getValue();
-                    int index = parentOp.getInputs().indexOf(ref);
-                    if (parentOp.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-                        AbstractLogicalOperator parentOpNext = (AbstractLogicalOperator) childrenToParents
-                                .get(parentOpRef).get(0).getValue();
-                        if (parentOpNext.isMap()) {
-                            index = parentOpNext.getInputs().indexOf(parentOpRef);
-                            parentOp = parentOpNext;
-                        }
-                    }
-
-                    ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator
-                            : projectOperator;
-                    if (parentOp.isMap()) {
-                        parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(childOp));
-                    } else {
-                        AbstractLogicalOperator exchg = new ExchangeOperator();
-                        exchg.setPhysicalOperator(new OneToOneExchangePOperator());
-                        exchg.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
-                        parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(exchg));
-                        context.computeAndSetTypeEnvironmentForOperator(exchg);
-                    }
-                    context.computeAndSetTypeEnvironmentForOperator(parentOp);
-                }
-            }
-            rewritten = true;
-        }
-        return rewritten;
-    }
-
-    private void genCandidates(IOptimizationContext context) throws AlgebricksException {
-        List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
-        while (equivalenceClasses.size() > 0) {
-            previousEquivalenceClasses.clear();
-            for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
-                List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
-                candidatesCopy.addAll(candidates);
-                previousEquivalenceClasses.add(candidatesCopy);
-            }
-            List<Mutable<ILogicalOperator>> currentLevelOpRefs = new ArrayList<Mutable<ILogicalOperator>>();
-            for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
-                if (candidates.size() > 0) {
-                    for (Mutable<ILogicalOperator> opRef : candidates) {
-                        List<Mutable<ILogicalOperator>> refs = childrenToParents.get(opRef);
-                        if (refs != null)
-                            currentLevelOpRefs.addAll(refs);
-                    }
-                }
-                if (currentLevelOpRefs.size() == 0)
-                    continue;
-                candidatesGrow(currentLevelOpRefs, candidates);
-            }
-            if (currentLevelOpRefs.size() == 0)
-                break;
-            prune(context);
-        }
-        if (equivalenceClasses.size() < 1 && previousEquivalenceClasses.size() > 0) {
-            equivalenceClasses.addAll(previousEquivalenceClasses);
-            prune(context);
-        }
-    }
-
-    private void topDownMaterialization(List<Mutable<ILogicalOperator>> tops) {
-        List<Mutable<ILogicalOperator>> candidates = new ArrayList<Mutable<ILogicalOperator>>();
-        List<Mutable<ILogicalOperator>> nextLevel = new ArrayList<Mutable<ILogicalOperator>>();
-        for (Mutable<ILogicalOperator> op : tops) {
-            for (Mutable<ILogicalOperator> opRef : op.getValue().getInputs()) {
-                List<Mutable<ILogicalOperator>> opRefList = childrenToParents.get(opRef);
-                if (opRefList == null) {
-                    opRefList = new ArrayList<Mutable<ILogicalOperator>>();
-                    childrenToParents.put(opRef, opRefList);
-                    nextLevel.add(opRef);
-                }
-                opRefList.add(op);
-            }
-            if (op.getValue().getInputs().size() == 0)
-                candidates.add(op);
-        }
-        if (equivalenceClasses.size() > 0) {
-            equivalenceClasses.get(0).addAll(candidates);
-        } else {
-            equivalenceClasses.add(candidates);
-        }
-        if (nextLevel.size() > 0) {
-            topDownMaterialization(nextLevel);
-        }
-    }
-
-    private void candidatesGrow(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) {
-        List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<Mutable<ILogicalOperator>>();
-        previousCandidates.addAll(candidates);
-        candidates.clear();
-        boolean validCandidate = false;
-        for (Mutable<ILogicalOperator> op : opList) {
-            List<Mutable<ILogicalOperator>> inputs = op.getValue().getInputs();
-            for (int i = 0; i < inputs.size(); i++) {
-                Mutable<ILogicalOperator> inputRef = inputs.get(i);
-                validCandidate = false;
-                for (Mutable<ILogicalOperator> candidate : previousCandidates) {
-                    // if current input is in candidates
-                    if (inputRef.getValue().equals(candidate.getValue())) {
-                        if (inputs.size() == 1) {
-                            validCandidate = true;
-                        } else {
-                            BitSet candidateInputBitMap = opToCandidateInputs.get(op);
-                            if (candidateInputBitMap == null) {
-                                candidateInputBitMap = new BitSet(inputs.size());
-                                opToCandidateInputs.put(op, candidateInputBitMap);
-                            }
-                            candidateInputBitMap.set(i);
-                            if (candidateInputBitMap.cardinality() == inputs.size()) {
-                                validCandidate = true;
-                            }
-                        }
-                        break;
-                    }
-                }
-            }
-            if (!validCandidate)
-                continue;
-            if (!candidates.contains(op))
-                candidates.add(op);
-        }
-    }
-
-    private void prune(IOptimizationContext context) throws AlgebricksException {
-        List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
-        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
-            List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
-            candidatesCopy.addAll(candidates);
-            previousEquivalenceClasses.add(candidatesCopy);
-        }
-        equivalenceClasses.clear();
-        for (List<Mutable<ILogicalOperator>> candidates : previousEquivalenceClasses) {
-            boolean[] reserved = new boolean[candidates.size()];
-            for (int i = 0; i < reserved.length; i++)
-                reserved[i] = false;
-            for (int i = candidates.size() - 1; i >= 0; i--) {
-                if (reserved[i] == false) {
-                    List<Mutable<ILogicalOperator>> equivalentClass = new ArrayList<Mutable<ILogicalOperator>>();
-                    ILogicalOperator candidate = candidates.get(i).getValue();
-                    equivalentClass.add(candidates.get(i));
-                    for (int j = i - 1; j >= 0; j--) {
-                        ILogicalOperator peer = candidates.get(j).getValue();
-                        if (IsomorphismUtilities.isOperatorIsomorphic(candidate, peer)) {
-                            reserved[i] = true;
-                            reserved[j] = true;
-                            equivalentClass.add(candidates.get(j));
-                        }
-                    }
-                    if (equivalentClass.size() > 1) {
-                        equivalenceClasses.add(equivalentClass);
-                        Collections.reverse(equivalentClass);
-                    }
-                }
-            }
-            for (int i = candidates.size() - 1; i >= 0; i--) {
-                if (!reserved[i]) {
-                    candidates.remove(i);
-                }
-            }
-        }
-    }
-
-    private boolean[] computeMaterilizationFlags(List<Mutable<ILogicalOperator>> group) {
-        lastUsedClusterId = 0;
-        for (Mutable<ILogicalOperator> root : roots) {
-            computeClusters(null, root, new MutableInt(++lastUsedClusterId));
-        }
-        boolean[] materializationFlags = new boolean[group.size()];
-        boolean worthMaterialization = worthMaterialization(group.get(0));
-        boolean requiresMaterialization;
-        // get clusterIds for each candidate in the group
-        List<Integer> groupClusterIds = new ArrayList<Integer>(group.size());
-        for (int i = 0; i < group.size(); i++) {
-            groupClusterIds.add(clusterMap.get(group.get(i)).getValue());
-        }
-        for (int i = group.size() - 1; i >= 0; i--) {
-            requiresMaterialization = requiresMaterialization(groupClusterIds, i);
-            if (requiresMaterialization && !worthMaterialization) {
-                group.remove(i);
-                groupClusterIds.remove(i);
-            }
-            materializationFlags[i] = requiresMaterialization;
-        }
-        if (group.size() < 2) {
-            group.clear();
-        }
-        // if does not worth materialization, the flags for the remaining candidates should be false
-        return worthMaterialization ? materializationFlags : new boolean[group.size()];
-    }
-
-    private boolean requiresMaterialization(List<Integer> groupClusterIds, int index) {
-        Integer clusterId = groupClusterIds.get(index);
-        BitSet blockingClusters = new BitSet();
-        getAllBlockingClusterIds(clusterId, blockingClusters);
-        if (!blockingClusters.isEmpty()) {
-            for (int i = 0; i < groupClusterIds.size(); i++) {
-                if (i == index) {
-                    continue;
-                }
-                if (blockingClusters.get(groupClusterIds.get(i))) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    private void getAllBlockingClusterIds(int clusterId, BitSet blockingClusters) {
-        BitSet waitFor = clusterWaitForMap.get(clusterId);
-        if (waitFor != null) {
-            for (int i = waitFor.nextSetBit(0); i >= 0; i = waitFor.nextSetBit(i + 1)) {
-                getAllBlockingClusterIds(i, blockingClusters);
-            }
-            blockingClusters.or(waitFor);
-        }
-    }
-
-    private void computeClusters(Mutable<ILogicalOperator> parentRef, Mutable<ILogicalOperator> opRef,
-            MutableInt currentClusterId) {
-        // only replicate operator has multiple outputs
-        int outputIndex = 0;
-        if (opRef.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) {
-            ReplicateOperator rop = (ReplicateOperator) opRef.getValue();
-            List<Mutable<ILogicalOperator>> outputs = rop.getOutputs();
-            for (outputIndex = 0; outputIndex < outputs.size(); outputIndex++) {
-                if (outputs.get(outputIndex).equals(parentRef)) {
-                    break;
-                }
-            }
-        }
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
-        Pair<int[], int[]> labels = aop.getPhysicalOperator().getInputOutputDependencyLabels(opRef.getValue());
-        List<Mutable<ILogicalOperator>> inputs = opRef.getValue().getInputs();
-        for (int i = 0; i < inputs.size(); i++) {
-            Mutable<ILogicalOperator> inputRef = inputs.get(i);
-            if (labels.second[outputIndex] == 1 && labels.first[i] == 0) { // 1 -> 0
-                if (labels.second.length == 1) {
-                    clusterMap.put(opRef, currentClusterId);
-                    // start a new cluster
-                    MutableInt newClusterId = new MutableInt(++lastUsedClusterId);
-                    computeClusters(opRef, inputRef, newClusterId);
-                    BitSet waitForList = clusterWaitForMap.get(currentClusterId.getValue());
-                    if (waitForList == null) {
-                        waitForList = new BitSet();
-                        clusterWaitForMap.put(currentClusterId.getValue(), waitForList);
-                    }
-                    waitForList.set(newClusterId.getValue());
-                }
-            } else { // 0 -> 0 and 1 -> 1
-                MutableInt prevClusterId = clusterMap.get(opRef);
-                if (prevClusterId == null || prevClusterId.getValue().equals(currentClusterId.getValue())) {
-                    clusterMap.put(opRef, currentClusterId);
-                    computeClusters(opRef, inputRef, currentClusterId);
-                } else {
-                    // merge prevClusterId and currentClusterId: update all the map entries that has currentClusterId to prevClusterId
-                    for (BitSet bs : clusterWaitForMap.values()) {
-                        if (bs.get(currentClusterId.getValue())) {
-                            bs.clear(currentClusterId.getValue());
-                            bs.set(prevClusterId.getValue());
-                        }
-                    }
-                    currentClusterId.setValue(prevClusterId.getValue());
-                }
-            }
-        }
-    }
-
-    protected boolean worthMaterialization(Mutable<ILogicalOperator> candidate) {
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) candidate.getValue();
-        if (aop.getPhysicalOperator().expensiveThanMaterialization()) {
-            return true;
-        }
-        List<Mutable<ILogicalOperator>> inputs = candidate.getValue().getInputs();
-        for (Mutable<ILogicalOperator> inputRef : inputs) {
-            if (worthMaterialization(inputRef)) {
-                return true;
-            }
-        }
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java
deleted file mode 100644
index 8f8e62b..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Factors out function expressions from each comparison function or similarity function in join condition by
- * assigning them to a variables, and replacing the function expressions with references to those variables.
- * Examples:
- * Plan with function expressions in comparison or similarity condition of join expression.
- * Generates one assign operator per extracted function expression.
- *
- * <pre>
- * Before plan:
- * 
- *   join ( eq( funcX($$1), funcX($$2) ) )
- * 
- * After plan:
- * 
- *   join (eq($$3,$$4))
- *   assign [$$4] <- [funcY($$2)]
- *   assign [$$3] <- [funcX($$1)]
- * </pre>
- */
-public class ExtractFunctionsFromJoinConditionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-
-        if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN
-                && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
-        ILogicalExpression expr = joinOp.getCondition().getValue();
-
-        return assignFunctionExpressions(joinOp, expr, context);
-
-    }
-
-    private boolean assignFunctionExpressions(AbstractLogicalOperator joinOp, ILogicalExpression expr,
-            IOptimizationContext context) throws AlgebricksException {
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
-        FunctionIdentifier fi = fexp.getFunctionIdentifier();
-
-        boolean modified = false;
-        if (fi.equals(AlgebricksBuiltinFunctions.AND) || fi.equals(AlgebricksBuiltinFunctions.OR)
-                || processArgumentsToFunction(fi)) {
-            for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
-                if (assignFunctionExpressions(joinOp, a.getValue(), context)) {
-                    modified = true;
-                }
-            }
-            return modified;
-        } else if (AlgebricksBuiltinFunctions.isComparisonFunction(fi) || isComparisonFunction(fi)) {
-            for (Mutable<ILogicalExpression> exprRef : fexp.getArguments()) {
-                if (exprRef.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                    LogicalVariable newVar = context.newVar();
-                    AssignOperator newAssign = new AssignOperator(newVar, new MutableObject<ILogicalExpression>(exprRef
-                            .getValue().cloneExpression()));
-                    newAssign.setExecutionMode(joinOp.getExecutionMode());
-
-                    // Place assign below joinOp.
-                    List<LogicalVariable> used = new ArrayList<LogicalVariable>();
-                    VariableUtilities.getUsedVariables(newAssign, used);
-
-                    Mutable<ILogicalOperator> leftBranchRef = joinOp.getInputs().get(0);
-                    ILogicalOperator leftBranch = leftBranchRef.getValue();
-                    List<LogicalVariable> leftBranchVariables = new ArrayList<LogicalVariable>();
-                    VariableUtilities.getLiveVariables(leftBranch, leftBranchVariables);
-                    if (leftBranchVariables.containsAll(used)) {
-                        // place assign on left branch
-                        newAssign.getInputs().add(new MutableObject<ILogicalOperator>(leftBranch));
-                        leftBranchRef.setValue(newAssign);
-                        modified = true;
-                    } else {
-                        Mutable<ILogicalOperator> rightBranchRef = joinOp.getInputs().get(1);
-                        ILogicalOperator rightBranch = rightBranchRef.getValue();
-                        List<LogicalVariable> rightBranchVariables = new ArrayList<LogicalVariable>();
-                        VariableUtilities.getLiveVariables(rightBranch, rightBranchVariables);
-                        if (rightBranchVariables.containsAll(used)) {
-                            // place assign on right branch
-                            newAssign.getInputs().add(new MutableObject<ILogicalOperator>(rightBranch));
-                            rightBranchRef.setValue(newAssign);
-                            modified = true;
-                        }
-                    }
-
-                    if (modified) {
-                        // Replace original expr with variable reference.
-                        exprRef.setValue(new VariableReferenceExpression(newVar));
-                        context.computeAndSetTypeEnvironmentForOperator(newAssign);
-                        context.computeAndSetTypeEnvironmentForOperator(joinOp);
-                    }
-                }
-            }
-            return modified;
-        } else {
-            return false;
-        }
-    }
-
-    protected boolean processArgumentsToFunction(FunctionIdentifier fi) {
-        return false;
-    }
-
-    protected boolean isComparisonFunction(FunctionIdentifier fi) {
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
deleted file mode 100644
index 2790166..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-
-/**
- * Needed only bc. current Hyrax operators require keys to be fields.
- */
-public class ExtractGbyExpressionsRule extends AbstractExtractExprRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-
-        if (context.checkIfInDontApplySet(this, op1)) {
-            return false;
-        }
-        context.addToDontApplySet(this, op1);
-        GroupByOperator g = (GroupByOperator) op1;
-        boolean r1 = gbyExprWasRewritten(g, context);
-        boolean r2 = decorExprWasRewritten(g, context);
-        boolean fired = r1 || r2;
-        if (fired) {
-            context.computeAndSetTypeEnvironmentForOperator(g);
-        }
-        return fired;
-    }
-
-    private boolean gbyExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
-        if (!gbyHasComplexExpr(g)) {
-            return false;
-        }
-        Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
-            ILogicalExpression expr = gbyPair.second.getValue();
-            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
-                gbyPair.second.setValue(new VariableReferenceExpression(v));
-            }
-        }
-        return true;
-    }
-
-    private boolean decorExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
-        if (!decorHasComplexExpr(g)) {
-            return false;
-        }
-        Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : g.getDecorList()) {
-            ILogicalExpression expr = decorPair.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
-                decorPair.second.setValue(new VariableReferenceExpression(v));
-            }
-        }
-        return true;
-    }
-
-    private boolean gbyHasComplexExpr(GroupByOperator g) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
-            if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private boolean decorHasComplexExpr(GroupByOperator g) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getDecorList()) {
-            if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
deleted file mode 100644
index 6cb01f2..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class FactorRedundantGroupAndDecorVarsRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        GroupByOperator gby = (GroupByOperator) op;
-        Map<LogicalVariable, LogicalVariable> varRhsToLhs = new HashMap<LogicalVariable, LogicalVariable>();
-        boolean gvChanged = factorRedundantRhsVars(gby.getGroupByList(), opRef, varRhsToLhs, context);
-        boolean dvChanged = factorRedundantRhsVars(gby.getDecorList(), opRef, varRhsToLhs, context);
-
-        return gvChanged || dvChanged;
-    }
-
-    private boolean factorRedundantRhsVars(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> veList,
-            Mutable<ILogicalOperator> opRef, Map<LogicalVariable, LogicalVariable> varRhsToLhs,
-            IOptimizationContext context) throws AlgebricksException {
-        varRhsToLhs.clear();
-        ListIterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = veList.listIterator();
-        boolean changed = false;
-        while (iter.hasNext()) {
-            Pair<LogicalVariable, Mutable<ILogicalExpression>> p = iter.next();
-            if (p.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                continue;
-            }
-            LogicalVariable v = GroupByOperator.getDecorVariable(p);
-            LogicalVariable lhs = varRhsToLhs.get(v);
-            if (lhs != null) {
-                if (p.first != null) {
-                    AssignOperator assign = new AssignOperator(p.first, new MutableObject<ILogicalExpression>(
-                            new VariableReferenceExpression(lhs)));
-                    ILogicalOperator op = opRef.getValue();
-                    assign.getInputs().add(new MutableObject<ILogicalOperator>(op));
-                    opRef.setValue(assign);
-                    context.computeAndSetTypeEnvironmentForOperator(assign);
-                }
-                iter.remove();
-                changed = true;
-            } else {
-                varRhsToLhs.put(v, p.first);
-            }
-        }
-        return changed;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
deleted file mode 100644
index 3530a5f..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class InferTypesRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        ILogicalOperator op = opRef.getValue();
-        if (context.getOutputTypeEnvironment(op) != null) {
-            return false;
-        }
-        context.computeAndSetTypeEnvironmentForOperator(op);
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
deleted file mode 100644
index 99a1fae..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.AbstractConstVarFunVisitor;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class InlineAssignIntoAggregateRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        boolean changed = false;
-        GroupByOperator gbyOp = (GroupByOperator) op;
-        for (ILogicalPlan p : gbyOp.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                if (inlined(r)) {
-                    changed = true;
-                }
-            }
-        }
-        return changed;
-    }
-
-    private boolean inlined(Mutable<ILogicalOperator> r) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-            return false;
-        }
-        AggregateOperator agg = (AggregateOperator) op1;
-        AssignOperator assign = (AssignOperator) op2;
-        VarExprSubstitution ves = new VarExprSubstitution(assign.getVariables(), assign.getExpressions());
-        for (Mutable<ILogicalExpression> exprRef : agg.getExpressions()) {
-            ILogicalExpression expr = exprRef.getValue();
-            Pair<Boolean, ILogicalExpression> p = expr.accept(ves, null);
-            if (p.first == true) {
-                exprRef.setValue(p.second);
-            }
-            // AbstractLogicalExpression ale = (AbstractLogicalExpression) expr;
-            // ale.accept(ves, null);
-        }
-        List<Mutable<ILogicalOperator>> op1InpList = op1.getInputs();
-        op1InpList.clear();
-        op1InpList.add(op2.getInputs().get(0));
-        return true;
-    }
-
-    private class VarExprSubstitution extends AbstractConstVarFunVisitor<Pair<Boolean, ILogicalExpression>, Void> {
-
-        private List<LogicalVariable> variables;
-        private List<Mutable<ILogicalExpression>> expressions;
-
-        public VarExprSubstitution(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
-            this.variables = variables;
-            this.expressions = expressions;
-        }
-
-        @Override
-        public Pair<Boolean, ILogicalExpression> visitConstantExpression(ConstantExpression expr, Void arg) {
-            return new Pair<Boolean, ILogicalExpression>(false, expr);
-        }
-
-        @Override
-        public Pair<Boolean, ILogicalExpression> visitFunctionCallExpression(AbstractFunctionCallExpression expr,
-                Void arg) throws AlgebricksException {
-            boolean changed = false;
-            for (Mutable<ILogicalExpression> eRef : expr.getArguments()) {
-                ILogicalExpression e = eRef.getValue();
-                Pair<Boolean, ILogicalExpression> p = e.accept(this, arg);
-                if (p.first) {
-                    eRef.setValue(p.second);
-                    changed = true;
-                }
-            }
-            return new Pair<Boolean, ILogicalExpression>(changed, expr);
-        }
-
-        @Override
-        public Pair<Boolean, ILogicalExpression> visitVariableReferenceExpression(VariableReferenceExpression expr,
-                Void arg) {
-            LogicalVariable v = expr.getVariableReference();
-            int idx = variables.indexOf(v);
-            if (idx < 0) {
-                return new Pair<Boolean, ILogicalExpression>(false, expr);
-            } else {
-                return new Pair<Boolean, ILogicalExpression>(true, expressions.get(idx).getValue());
-            }
-
-        }
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java
deleted file mode 100644
index 9ae94ba..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-/**
- * Inlines variables that are referenced exactly once.
- * 
- * Preconditions/Assumptions:
- * Assumes no projects are in the plan.
- * 
- * Example assuming variable $$3 is referenced exactly once.
- * 
- * Before plan:
- * select (funcA($$3))
- *   ...
- *     assign [$$3] <- [field-access($$0, 1)]
- * 
- * After plan:
- * select (funcA(field-access($$0, 1))
- *   ...
- *     assign [] <- []
- */
-public class InlineSingleReferenceVariablesRule extends InlineVariablesRule {
-
-    // Maps from variable to a list of operators using that variable.
-    protected Map<LogicalVariable, List<ILogicalOperator>> usedVarsMap = new HashMap<LogicalVariable, List<ILogicalOperator>>();
-    protected List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
-    
-    @Override
-    protected void prepare(IOptimizationContext context) {
-        super.prepare(context);
-        usedVarsMap.clear();
-        usedVars.clear();
-    }
-    
-    @Override
-    protected boolean performFinalAction() throws AlgebricksException {
-        boolean modified = false;
-        for (Map.Entry<LogicalVariable, List<ILogicalOperator>> entry : usedVarsMap.entrySet()) {
-            // Perform replacement only if variable is referenced a single time.
-            if (entry.getValue().size() == 1) {
-                ILogicalOperator op = entry.getValue().get(0);
-                if (!op.requiresVariableReferenceExpressions()) {
-                    inlineVisitor.setOperator(op);
-                    inlineVisitor.setTargetVariable(entry.getKey());
-                    if (op.acceptExpressionTransform(inlineVisitor)) {
-                        modified = true;
-                    }
-                    inlineVisitor.setTargetVariable(null);
-                }         
-            }
-        }
-        return modified;
-    }
-
-    @Override
-    protected boolean performBottomUpAction(AbstractLogicalOperator op) throws AlgebricksException {
-        usedVars.clear();
-        VariableUtilities.getUsedVariables(op, usedVars);
-        for (LogicalVariable var : usedVars) {
-            List<ILogicalOperator> opsUsingVar = usedVarsMap.get(var);
-            if (opsUsingVar == null) {
-                opsUsingVar = new ArrayList<ILogicalOperator>();
-                usedVarsMap.put(var, opsUsingVar);
-            }
-            opsUsingVar.add(op);
-        }
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
deleted file mode 100644
index 6390efc..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Replaces variable reference expressions with their assigned function-call expression where applicable
- * (some variables are generated by datasources).
- * Inlining variables may enable other optimizations by allowing selects and assigns to be moved
- * (e.g., a select may be pushed into a join to enable an efficient physical join operator).
- * Preconditions/Assumptions:
- * Assumes no projects are in the plan. Only inlines variables whose assigned expression is a function call
- * (i.e., this rule ignores right-hand side constants and other variable references expressions
- * Postconditions/Examples:
- * All qualifying variables have been inlined.
- * Example (simplified):
- * Before plan:
- * select <- [$$1 < $$2 + $$0]
- * assign [$$2] <- [funcZ() + $$0]
- * assign [$$0, $$1] <- [funcX(), funcY()]
- * After plan:
- * select <- [funcY() < funcZ() + funcX() + funcX()]
- * assign [$$2] <- [funcZ() + funcX()]
- * assign [$$0, $$1] <- [funcX(), funcY()]
- */
-public class InlineVariablesRule implements IAlgebraicRewriteRule {
-
-    // Map of variables that could be replaced by their producing expression.
-    // Populated during the top-down sweep of the plan.
-    protected Map<LogicalVariable, ILogicalExpression> varAssignRhs = new HashMap<LogicalVariable, ILogicalExpression>();
-
-    // Visitor for replacing variable reference expressions with their originating expression.
-    protected InlineVariablesVisitor inlineVisitor = new InlineVariablesVisitor(varAssignRhs);
-
-    // Set of FunctionIdentifiers that we should not inline.
-    protected Set<FunctionIdentifier> doNotInlineFuncs = new HashSet<FunctionIdentifier>();
-
-    protected boolean hasRun = false;
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        if (hasRun) {
-            return false;
-        }
-        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
-            return false;
-        }
-        prepare(context);
-        boolean modified = inlineVariables(opRef, context);
-        if (performFinalAction()) {
-            modified = true;
-        }
-        hasRun = true;
-        return modified;
-    }
-
-    protected void prepare(IOptimizationContext context) {
-        varAssignRhs.clear();
-        inlineVisitor.setContext(context);
-    }
-
-    protected boolean performBottomUpAction(AbstractLogicalOperator op) throws AlgebricksException {
-        // Only inline variables in operators that can deal with arbitrary expressions.
-        if (!op.requiresVariableReferenceExpressions()) {
-            inlineVisitor.setOperator(op);
-            return op.acceptExpressionTransform(inlineVisitor);
-        }
-        return false;
-    }
-
-    protected boolean performFinalAction() throws AlgebricksException {
-        return false;
-    }
-
-    protected boolean inlineVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-
-        // Update mapping from variables to expressions during top-down traversal.
-        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-            AssignOperator assignOp = (AssignOperator) op;
-            List<LogicalVariable> vars = assignOp.getVariables();
-            List<Mutable<ILogicalExpression>> exprs = assignOp.getExpressions();
-            for (int i = 0; i < vars.size(); i++) {
-                ILogicalExpression expr = exprs.get(i).getValue();
-                // Ignore functions that are either in the doNotInline set or are non-functional               
-                if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                    AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-                    if (doNotInlineFuncs.contains(funcExpr.getFunctionIdentifier()) || !funcExpr.isFunctional()) {
-                        continue;
-                    }
-                }
-                varAssignRhs.put(vars.get(i), exprs.get(i).getValue());
-            }
-        }
-
-        // Descend into children removing projects on the way.
-        boolean modified = false;
-        for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
-            if (inlineVariables(inputOpRef, context)) {
-                modified = true;
-            }
-        }
-
-        // Descend into subplan
-        if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
-            ALogicalPlanImpl subPlan = (ALogicalPlanImpl) ((SubplanOperator) op).getNestedPlans().get(0);
-            Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0);
-            if (inlineVariables(subPlanRootOpRef, context)) {
-                modified = true;
-            }
-        }
-
-        if (performBottomUpAction(op)) {
-            modified = true;
-        }
-
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(op);
-            context.addToDontApplySet(this, op);
-            // Re-enable rules that we may have already tried. They could be applicable now after inlining.
-            context.removeFromAlreadyCompared(opRef.getValue());
-        }
-
-        return modified;
-    }
-
-    protected class InlineVariablesVisitor implements ILogicalExpressionReferenceTransform {
-
-        private final Map<LogicalVariable, ILogicalExpression> varAssignRhs;
-        private final Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
-        private final List<LogicalVariable> rhsUsedVars = new ArrayList<LogicalVariable>();
-        private ILogicalOperator op;
-        private IOptimizationContext context;
-        // If set, only replace this variable reference.
-        private LogicalVariable targetVar;
-
-        public InlineVariablesVisitor(Map<LogicalVariable, ILogicalExpression> varAssignRhs) {
-            this.varAssignRhs = varAssignRhs;
-        }
-
-        public void setTargetVariable(LogicalVariable targetVar) {
-            this.targetVar = targetVar;
-        }
-
-        public void setContext(IOptimizationContext context) {
-            this.context = context;
-        }
-
-        public void setOperator(ILogicalOperator op) throws AlgebricksException {
-            this.op = op;
-            liveVars.clear();
-        }
-
-        @Override
-        public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
-            ILogicalExpression e = exprRef.getValue();
-            switch (((AbstractLogicalExpression) e).getExpressionTag()) {
-                case VARIABLE: {
-                    LogicalVariable var = ((VariableReferenceExpression) e).getVariableReference();
-                    // Restrict replacement to targetVar if it has been set.
-                    if (targetVar != null && var != targetVar) {
-                        return false;
-                    }
-
-                    // Make sure has not been excluded from inlining.
-                    if (context.shouldNotBeInlined(var)) {
-                        return false;
-                    }
-
-                    ILogicalExpression rhs = varAssignRhs.get(var);
-                    if (rhs == null) {
-                        // Variable was not produced by an assign.
-                        return false;
-                    }
-
-                    // Make sure used variables from rhs are live.
-                    if (liveVars.isEmpty()) {
-                        VariableUtilities.getLiveVariables(op, liveVars);
-                    }
-                    rhsUsedVars.clear();
-                    rhs.getUsedVariables(rhsUsedVars);
-                    for (LogicalVariable rhsUsedVar : rhsUsedVars) {
-                        if (!liveVars.contains(rhsUsedVar)) {
-                            return false;
-                        }
-                    }
-
-                    // Replace variable reference with a clone of the rhs expr.
-                    exprRef.setValue(rhs.cloneExpression());
-                    return true;
-                }
-                case FUNCTION_CALL: {
-                    AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
-                    boolean modified = false;
-                    for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
-                        if (transform(arg)) {
-                            modified = true;
-                        }
-                    }
-                    return modified;
-                }
-                default: {
-                    return false;
-                }
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
deleted file mode 100644
index c5aee3c..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.Iterator;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class InsertOuterJoinRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
-        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op0;
-
-        Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
-        ILogicalPlan p = null;
-        while (plansIter.hasNext()) {
-            p = plansIter.next();
-        }
-        if (p == null) {
-            return false;
-        }
-        if (p.getRoots().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
-        Mutable<ILogicalOperator> opUnder = subplan.getInputs().get(0);
-
-        if (OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) opUnder.getValue())) {
-            return false;
-        }
-
-        switch (op1.getOperatorTag()) {
-            case INNERJOIN: {
-                InnerJoinOperator join = (InnerJoinOperator) op1;
-                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
-                Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
-                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
-                if (ntsRef == null) {
-                    ntsRef = getNtsAtEndOfPipeline(rightRef);
-                    if (ntsRef == null) {
-                        return false;
-                    } else {
-                        Mutable<ILogicalOperator> t = leftRef;
-                        leftRef = rightRef;
-                        rightRef = t;
-                    }
-                }
-                ntsRef.setValue(opUnder.getValue());
-                LeftOuterJoinOperator loj = new LeftOuterJoinOperator(join.getCondition());
-                loj.getInputs().add(leftRef);
-                loj.getInputs().add(rightRef);
-                opRef.setValue(loj);
-                context.computeAndSetTypeEnvironmentForOperator(loj);
-                return true;
-            }
-            case LEFTOUTERJOIN: {
-                LeftOuterJoinOperator join = (LeftOuterJoinOperator) op1;
-                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
-                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
-                if (ntsRef == null) {
-                    return false;
-                }
-                ntsRef.setValue(opUnder.getValue());
-                opRef.setValue(join);
-                context.computeAndSetTypeEnvironmentForOperator(join);
-                return true;
-            }
-            default: {
-                return false;
-            }
-        }
-    }
-
-    private Mutable<ILogicalOperator> getNtsAtEndOfPipeline(Mutable<ILogicalOperator> opRef) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            return opRef;
-        }
-        if (op.getInputs().size() != 1) {
-            return null;
-        }
-        return getNtsAtEndOfPipeline(op.getInputs().get(0));
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
deleted file mode 100644
index 80b8815..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class InsertProjectBeforeUnionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        return false;
-    }
-
-    /**
-     * When the input schema to WriteOperator is different from the output
-     * schema in terms of variable order, add a project operator to get the
-     * write order
-     */
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
-            return false;
-        }
-        UnionAllOperator opUnion = (UnionAllOperator) op;
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = opUnion.getVariableMappings();
-        ArrayList<LogicalVariable> usedVariablesFromOne = new ArrayList<LogicalVariable>();
-        ArrayList<LogicalVariable> usedVariablesFromTwo = new ArrayList<LogicalVariable>();
-
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple : varMap) {
-            usedVariablesFromOne.add(triple.first);
-            usedVariablesFromTwo.add(triple.second);
-        }
-
-        ArrayList<LogicalVariable> inputSchemaOne = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(opUnion.getInputs().get(0).getValue(), inputSchemaOne);
-
-        ArrayList<LogicalVariable> inputSchemaTwo = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(opUnion.getInputs().get(1).getValue(), inputSchemaTwo);
-
-        boolean rewritten = false;
-        if (!isIdentical(usedVariablesFromOne, inputSchemaOne)) {
-            insertProjectOperator(opUnion, 0, usedVariablesFromOne, context);
-            rewritten = true;
-        }
-        if (!isIdentical(usedVariablesFromTwo, inputSchemaTwo)) {
-            insertProjectOperator(opUnion, 1, usedVariablesFromTwo, context);
-            rewritten = true;
-        }
-        return rewritten;
-    }
-
-    private void insertProjectOperator(UnionAllOperator opUnion, int branch, ArrayList<LogicalVariable> usedVariables,
-            IOptimizationContext context) throws AlgebricksException {
-        ProjectOperator projectOp = new ProjectOperator(usedVariables);
-        ILogicalOperator parentOp = opUnion.getInputs().get(branch).getValue();
-        projectOp.getInputs().add(new MutableObject<ILogicalOperator>(parentOp));
-        opUnion.getInputs().get(branch).setValue(projectOp);
-        projectOp.setPhysicalOperator(new StreamProjectPOperator());
-        context.computeAndSetTypeEnvironmentForOperator(projectOp);
-        context.computeAndSetTypeEnvironmentForOperator(parentOp);
-    }
-
-    private boolean isIdentical(List<LogicalVariable> finalSchema, List<LogicalVariable> inputSchema)
-            throws AlgebricksException {
-        int finalSchemaSize = finalSchema.size();
-        int inputSchemaSize = inputSchema.size();
-        if (finalSchemaSize != inputSchemaSize) {
-            return false;
-        }
-        for (int i = 0; i < finalSchemaSize; i++) {
-            LogicalVariable var1 = finalSchema.get(i);
-            LogicalVariable var2 = inputSchema.get(i);
-            if (!var1.equals(var2))
-                return false;
-        }
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
deleted file mode 100644
index 51f2025..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class IntroHashPartitionMergeExchange implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getPhysicalOperator() == null
-                || (op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_EXCHANGE && op1
-                        .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE)) {
-            return false;
-        }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-        if (op2.getPhysicalOperator() == null
-                || op2.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.SORT_MERGE_EXCHANGE) {
-            return false;
-        }
-        if (op1.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE) {
-            // if it is a hash_partition_merge_exchange, the sort_merge_exchange can be simply removed
-            op1.getInputs().get(0).setValue(op2.getInputs().get(0).getValue());
-            op1.computeDeliveredPhysicalProperties(context);
-            return true;
-        }
-        HashPartitionExchangePOperator hpe = (HashPartitionExchangePOperator) op1.getPhysicalOperator();
-        SortMergeExchangePOperator sme = (SortMergeExchangePOperator) op2.getPhysicalOperator();
-        List<OrderColumn> ocList = new ArrayList<OrderColumn>();
-        for (OrderColumn oc : sme.getSortColumns()) {
-            ocList.add(oc);
-        }
-        HashPartitionMergeExchangePOperator hpme = new HashPartitionMergeExchangePOperator(ocList, hpe.getHashFields(),
-                hpe.getDomain());
-        op1.setPhysicalOperator(hpme);
-        op1.getInputs().get(0).setValue(op2.getInputs().get(0).getValue());
-        op1.computeDeliveredPhysicalProperties(context);
-        return true;
-    }
-
-}

[08/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
new file mode 100644
index 0000000..41cf470
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
@@ -0,0 +1,436 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Factors out common sub-expressions by assigning them to a variables, and replacing the common sub-expressions with references to those variables.
+ * Preconditions/Assumptions:
+ * Assumes no projects are in the plan. This rule ignores variable reference expressions and constants (other rules deal with those separately).
+ * Postconditions/Examples:
+ * Plan with extracted sub-expressions. Generates one assign operator per extracted expression.
+ * Example 1 - Simple Arithmetic Example (simplified)
+ * Before plan:
+ * assign [$$1] <- [5 + 6 - 10]
+ * assign [$$0] <- [5 + 6 + 30]
+ * After plan:
+ * assign [$$1] <- [$$5 - 10]
+ * assign [$$0] <- [$$5 + 30]
+ * assign [$$5] <- [5 + 6]
+ * Example 2 - Cleaning up 'Distinct By' (simplified)
+ * Before plan: (notice how $$0 is not live after the distinct)
+ * assign [$$3] <- [field-access($$0, 1)]
+ * distinct ([%0->$$5])
+ * assign [$$5] <- [field-access($$0, 1)]
+ * unnest $$0 <- [scan-dataset]
+ * After plan: (notice how the issue of $$0 is fixed)
+ * assign [$$3] <- [$$5]
+ * distinct ([$$5])
+ * assign [$$5] <- [field-access($$0, 1)]
+ * unnest $$0 <- [scan-dataset]
+ * Example 3 - Pulling Common Expressions Above Joins (simplified)
+ * Before plan:
+ * assign [$$9] <- funcZ(funcY($$8))
+ * join (funcX(funcY($$8)))
+ * After plan:
+ * assign [$$9] <- funcZ($$10))
+ * select (funcX($$10))
+ * assign [$$10] <- [funcY($$8)]
+ * join (TRUE)
+ */
+public class ExtractCommonExpressionsRule implements IAlgebraicRewriteRule {
+
+    private final List<ILogicalExpression> originalAssignExprs = new ArrayList<ILogicalExpression>();
+
+    private final CommonExpressionSubstitutionVisitor substVisitor = new CommonExpressionSubstitutionVisitor();
+    private final Map<ILogicalExpression, ExprEquivalenceClass> exprEqClassMap = new HashMap<ILogicalExpression, ExprEquivalenceClass>();
+
+    // Set of operators for which common subexpression elimination should not be performed.
+    private static final Set<LogicalOperatorTag> ignoreOps = new HashSet<LogicalOperatorTag>();
+    static {
+        ignoreOps.add(LogicalOperatorTag.UNNEST);
+        ignoreOps.add(LogicalOperatorTag.UNNEST_MAP);
+        ignoreOps.add(LogicalOperatorTag.ORDER);
+        ignoreOps.add(LogicalOperatorTag.PROJECT);
+        ignoreOps.add(LogicalOperatorTag.AGGREGATE);
+        ignoreOps.add(LogicalOperatorTag.RUNNINGAGGREGATE);
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        exprEqClassMap.clear();
+        substVisitor.setContext(context);
+        boolean modified = removeCommonExpressions(opRef, context);
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
+        }
+        return modified;
+    }
+
+    private void updateEquivalenceClassMap(LogicalVariable lhs, Mutable<ILogicalExpression> rhsExprRef,
+            ILogicalExpression rhsExpr, ILogicalOperator op) {
+        ExprEquivalenceClass exprEqClass = exprEqClassMap.get(rhsExpr);
+        if (exprEqClass == null) {
+            exprEqClass = new ExprEquivalenceClass(op, rhsExprRef);
+            exprEqClassMap.put(rhsExpr, exprEqClass);
+        }
+        exprEqClass.setVariable(lhs);
+    }
+
+    private boolean removeCommonExpressions(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+            return false;
+        }
+
+        boolean modified = false;
+        // Recurse into children.
+        for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+            if (removeCommonExpressions(inputOpRef, context)) {
+                modified = true;
+            }
+        }
+
+        // TODO: Deal with replicate properly. Currently, we just clear the expr equivalence map, since we want to avoid incorrect expression replacement
+        // (the resulting new variables should be assigned live below a replicate).
+        if (op.getOperatorTag() == LogicalOperatorTag.REPLICATE) {
+            exprEqClassMap.clear();
+            return modified;
+        }
+        // Exclude these operators.
+        if (ignoreOps.contains(op.getOperatorTag())) {
+            return modified;
+        }
+
+        // Remember a copy of the original assign expressions, so we can add them to the equivalence class map
+        // after replacing expressions within the assign operator itself.
+        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+            AssignOperator assignOp = (AssignOperator) op;
+            originalAssignExprs.clear();
+            int numVars = assignOp.getVariables().size();
+            for (int i = 0; i < numVars; i++) {
+                Mutable<ILogicalExpression> exprRef = assignOp.getExpressions().get(i);
+                ILogicalExpression expr = exprRef.getValue();
+                originalAssignExprs.add(expr.cloneExpression());
+            }
+        }
+
+        // Perform common subexpression elimination.
+        substVisitor.setOperator(op);
+        if (op.acceptExpressionTransform(substVisitor)) {
+            modified = true;
+        }
+
+        // Update equivalence class map.
+        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+            AssignOperator assignOp = (AssignOperator) op;
+            int numVars = assignOp.getVariables().size();
+            for (int i = 0; i < numVars; i++) {
+                Mutable<ILogicalExpression> exprRef = assignOp.getExpressions().get(i);
+                ILogicalExpression expr = exprRef.getValue();
+                if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE
+                        || expr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+                    continue;
+                }
+                // Update equivalence class map.
+                LogicalVariable lhs = assignOp.getVariables().get(i);
+                updateEquivalenceClassMap(lhs, exprRef, exprRef.getValue(), op);
+
+                // Update equivalence class map with original assign expression.
+                updateEquivalenceClassMap(lhs, exprRef, originalAssignExprs.get(i), op);
+            }
+        }
+
+        // TODO: For now do not perform replacement in nested plans
+        // due to the complication of figuring out whether the firstOp in an equivalence class is within a subplan, 
+        // and the resulting variable will not be visible to the outside.
+        // Since subplans should be eliminated in most cases, this behavior is acceptable for now.
+        /*
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
+                    if (removeCommonExpressions(rootRef, context)) {
+                        modified = true;
+                    }
+                }
+            }
+        }
+        */
+
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(op);
+            context.addToDontApplySet(this, op);
+        }
+        return modified;
+    }
+
+    private class CommonExpressionSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
+
+        private IOptimizationContext context;
+        private ILogicalOperator op;
+
+        public void setContext(IOptimizationContext context) {
+            this.context = context;
+        }
+
+        public void setOperator(ILogicalOperator op) throws AlgebricksException {
+            this.op = op;
+        }
+
+        @Override
+        public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+            AbstractLogicalExpression expr = (AbstractLogicalExpression) exprRef.getValue();
+            boolean modified = false;
+            ExprEquivalenceClass exprEqClass = exprEqClassMap.get(expr);
+            if (exprEqClass != null) {
+                // Replace common subexpression with existing variable. 
+                if (exprEqClass.variableIsSet()) {
+                    Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
+                    List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getLiveVariables(op, liveVars);
+                    VariableUtilities.getUsedVariables(op, usedVars);
+                    // Check if the replacing variable is live at this op.
+                    // However, if the op is already using variables that are not live, then a replacement may enable fixing the plan.
+                    // This behavior is necessary to, e.g., properly deal with distinct by.
+                    // Also just replace the expr if we are replacing common exprs from within the same operator.
+                    if (liveVars.contains(exprEqClass.getVariable()) || !liveVars.containsAll(usedVars)
+                            || op == exprEqClass.getFirstOperator()) {
+                        exprRef.setValue(new VariableReferenceExpression(exprEqClass.getVariable()));
+                        // Do not descend into children since this expr has been completely replaced.
+                        return true;
+                    }
+                } else {
+                    if (expr.isFunctional() && assignCommonExpression(exprEqClass, expr)) {
+                        //re-obtain the live vars after rewriting in the method called in the if condition
+                        Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
+                        VariableUtilities.getLiveVariables(op, liveVars);
+                        //rewrite only when the variable is live
+                        if (liveVars.contains(exprEqClass.getVariable())) {
+                            exprRef.setValue(new VariableReferenceExpression(exprEqClass.getVariable()));
+                            // Do not descend into children since this expr has been completely replaced.
+                            return true;
+                        }
+                    }
+                }
+            } else {
+                if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE
+                        && expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+                    exprEqClass = new ExprEquivalenceClass(op, exprRef);
+                    exprEqClassMap.put(expr, exprEqClass);
+                }
+            }
+
+            // Descend into function arguments.
+            if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+                for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
+                    if (transform(arg)) {
+                        modified = true;
+                    }
+                }
+            }
+            return modified;
+        }
+
+        private boolean assignCommonExpression(ExprEquivalenceClass exprEqClass, ILogicalExpression expr)
+                throws AlgebricksException {
+            AbstractLogicalOperator firstOp = (AbstractLogicalOperator) exprEqClass.getFirstOperator();
+            Mutable<ILogicalExpression> firstExprRef = exprEqClass.getFirstExpression();
+            if (firstOp.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+                    || firstOp.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+                // Do not extract common expressions from within the same join operator.
+                if (firstOp == op) {
+                    return false;
+                }
+                AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) firstOp;
+                Mutable<ILogicalExpression> joinCond = joinOp.getCondition();
+                ILogicalExpression enclosingExpr = getEnclosingExpression(joinCond, firstExprRef.getValue());
+                if (enclosingExpr == null) {
+                    // No viable enclosing expression that we can pull out from the join.
+                    return false;
+                }
+                // Place a Select operator beneath op that contains the enclosing expression.
+                SelectOperator selectOp = new SelectOperator(new MutableObject<ILogicalExpression>(enclosingExpr),
+                        false, null);
+                selectOp.getInputs().add(new MutableObject<ILogicalOperator>(op.getInputs().get(0).getValue()));
+                op.getInputs().get(0).setValue(selectOp);
+                // Set firstOp to be the select below op, since we want to assign the common subexpr there.
+                firstOp = (AbstractLogicalOperator) selectOp;
+            } else if (firstOp.getInputs().size() > 1) {
+                // Bail for any non-join operator with multiple inputs.
+                return false;
+            }
+            LogicalVariable newVar = context.newVar();
+            AssignOperator newAssign = new AssignOperator(newVar, new MutableObject<ILogicalExpression>(firstExprRef
+                    .getValue().cloneExpression()));
+            // Place assign below firstOp.
+            newAssign.getInputs().add(new MutableObject<ILogicalOperator>(firstOp.getInputs().get(0).getValue()));
+            newAssign.setExecutionMode(firstOp.getExecutionMode());
+            firstOp.getInputs().get(0).setValue(newAssign);
+            // Replace original expr with variable reference, and set var in expression equivalence class.
+            firstExprRef.setValue(new VariableReferenceExpression(newVar));
+            exprEqClass.setVariable(newVar);
+            context.computeAndSetTypeEnvironmentForOperator(newAssign);
+            context.computeAndSetTypeEnvironmentForOperator(firstOp);
+            return true;
+        }
+
+        private ILogicalExpression getEnclosingExpression(Mutable<ILogicalExpression> conditionExprRef,
+                ILogicalExpression commonSubExpr) {
+            ILogicalExpression conditionExpr = conditionExprRef.getValue();
+            if (conditionExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+                return null;
+            }
+            if (isEqJoinCondition(commonSubExpr)) {
+                // Do not eliminate the common expression if we could use it for an equi-join.
+                return null;
+            }
+            AbstractFunctionCallExpression conditionFuncExpr = (AbstractFunctionCallExpression) conditionExpr;
+            // Boolean expression that encloses the common subexpression.
+            ILogicalExpression enclosingBoolExpr = null;
+            // We are not dealing with arbitrarily nested and/or expressions here.
+            FunctionIdentifier funcIdent = conditionFuncExpr.getFunctionIdentifier();
+            if (funcIdent.equals(AlgebricksBuiltinFunctions.AND) || funcIdent.equals(AlgebricksBuiltinFunctions.OR)) {
+                Iterator<Mutable<ILogicalExpression>> argIter = conditionFuncExpr.getArguments().iterator();
+                while (argIter.hasNext()) {
+                    Mutable<ILogicalExpression> argRef = argIter.next();
+                    if (containsExpr(argRef.getValue(), commonSubExpr)) {
+                        enclosingBoolExpr = argRef.getValue();
+                        // Remove the enclosing expression from the argument list.
+                        // We are going to pull it out into a new select operator.
+                        argIter.remove();
+                        break;
+                    }
+                }
+                // If and/or only has a single argument left, pull it out and remove the and/or function.
+                if (conditionFuncExpr.getArguments().size() == 1) {
+                    conditionExprRef.setValue(conditionFuncExpr.getArguments().get(0).getValue());
+                }
+            } else {
+                if (!containsExpr(conditionExprRef.getValue(), commonSubExpr)) {
+                    return null;
+                }
+                enclosingBoolExpr = conditionFuncExpr;
+                // Replace the enclosing expression with TRUE.
+                conditionExprRef.setValue(ConstantExpression.TRUE);
+            }
+            return enclosingBoolExpr;
+        }
+    }
+
+    private boolean containsExpr(ILogicalExpression expr, ILogicalExpression searchExpr) {
+        if (expr == searchExpr) {
+            return true;
+        }
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+        for (Mutable<ILogicalExpression> argRef : funcExpr.getArguments()) {
+            if (containsExpr(argRef.getValue(), searchExpr)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean isEqJoinCondition(ILogicalExpression expr) {
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+        if (funcExpr.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.EQ)) {
+            ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
+            ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
+            if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE
+                    && arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private final class ExprEquivalenceClass {
+        // First operator in which expression is used.
+        private final ILogicalOperator firstOp;
+
+        // Reference to expression in first op.
+        private final Mutable<ILogicalExpression> firstExprRef;
+
+        // Variable that this expression has been assigned to.
+        private LogicalVariable var;
+
+        public ExprEquivalenceClass(ILogicalOperator firstOp, Mutable<ILogicalExpression> firstExprRef) {
+            this.firstOp = firstOp;
+            this.firstExprRef = firstExprRef;
+        }
+
+        public ILogicalOperator getFirstOperator() {
+            return firstOp;
+        }
+
+        public Mutable<ILogicalExpression> getFirstExpression() {
+            return firstExprRef;
+        }
+
+        public void setVariable(LogicalVariable var) {
+            this.var = var;
+        }
+
+        public LogicalVariable getVariable() {
+            return var;
+        }
+
+        public boolean variableIsSet() {
+            return var != null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
new file mode 100644
index 0000000..07621ce
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
@@ -0,0 +1,502 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule {
+
+    private final HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> childrenToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
+    private final List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
+    private final List<List<Mutable<ILogicalOperator>>> equivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+    private final HashMap<Mutable<ILogicalOperator>, BitSet> opToCandidateInputs = new HashMap<Mutable<ILogicalOperator>, BitSet>();
+    private final HashMap<Mutable<ILogicalOperator>, MutableInt> clusterMap = new HashMap<Mutable<ILogicalOperator>, MutableInt>();
+    private final HashMap<Integer, BitSet> clusterWaitForMap = new HashMap<Integer, BitSet>();
+    private int lastUsedClusterId = 0;
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT
+                && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) {
+            return false;
+        }
+        if (!roots.contains(op))
+            roots.add(new MutableObject<ILogicalOperator>(op));
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT
+                && op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT) {
+            return false;
+        }
+        boolean rewritten = false;
+        boolean changed = false;
+        if (roots.size() > 0) {
+            do {
+                changed = false;
+                // applying the rewriting until fixpoint
+                topDownMaterialization(roots);
+                genCandidates(context);
+                removeTrivialShare();
+                if (equivalenceClasses.size() > 0)
+                    changed = rewrite(context);
+                if (!rewritten)
+                    rewritten = changed;
+                equivalenceClasses.clear();
+                childrenToParents.clear();
+                opToCandidateInputs.clear();
+                clusterMap.clear();
+                clusterWaitForMap.clear();
+                lastUsedClusterId = 0;
+            } while (changed);
+            roots.clear();
+        }
+        return rewritten;
+    }
+
+    private void removeTrivialShare() {
+        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+            for (int i = candidates.size() - 1; i >= 0; i--) {
+                Mutable<ILogicalOperator> opRef = candidates.get(i);
+                AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
+                if (aop.getOperatorTag() == LogicalOperatorTag.EXCHANGE)
+                    aop = (AbstractLogicalOperator) aop.getInputs().get(0).getValue();
+                if (aop.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE)
+                    candidates.remove(i);
+            }
+        }
+        for (int i = equivalenceClasses.size() - 1; i >= 0; i--)
+            if (equivalenceClasses.get(i).size() < 2)
+                equivalenceClasses.remove(i);
+    }
+
+    private boolean rewrite(IOptimizationContext context) throws AlgebricksException {
+        boolean changed = false;
+        for (List<Mutable<ILogicalOperator>> members : equivalenceClasses) {
+            if (rewriteForOneEquivalentClass(members, context))
+                changed = true;
+        }
+        return changed;
+    }
+
+    private boolean rewriteForOneEquivalentClass(List<Mutable<ILogicalOperator>> members, IOptimizationContext context)
+            throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> group = new ArrayList<Mutable<ILogicalOperator>>();
+        boolean rewritten = false;
+        while (members.size() > 0) {
+            group.clear();
+            Mutable<ILogicalOperator> candidate = members.remove(members.size() - 1);
+            group.add(candidate);
+            for (int i = members.size() - 1; i >= 0; i--) {
+                Mutable<ILogicalOperator> peer = members.get(i);
+                if (IsomorphismUtilities.isOperatorIsomorphic(candidate.getValue(), peer.getValue())) {
+                    group.add(peer);
+                    members.remove(i);
+                }
+            }
+            boolean[] materializationFlags = computeMaterilizationFlags(group);
+            if (group.isEmpty()) {
+                continue;
+            }
+            candidate = group.get(0);
+            ReplicateOperator rop = new ReplicateOperator(group.size(), materializationFlags);
+            rop.setPhysicalOperator(new ReplicatePOperator());
+            rop.setExecutionMode(ExecutionMode.PARTITIONED);
+            Mutable<ILogicalOperator> ropRef = new MutableObject<ILogicalOperator>(rop);
+            AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getValue();
+            List<Mutable<ILogicalOperator>> originalCandidateParents = childrenToParents.get(candidate);
+
+            if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+                rop.getInputs().add(candidate);
+            } else {
+                AbstractLogicalOperator beforeExchange = new ExchangeOperator();
+                beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator());
+                Mutable<ILogicalOperator> beforeExchangeRef = new MutableObject<ILogicalOperator>(beforeExchange);
+                beforeExchange.getInputs().add(candidate);
+                context.computeAndSetTypeEnvironmentForOperator(beforeExchange);
+                rop.getInputs().add(beforeExchangeRef);
+            }
+            context.computeAndSetTypeEnvironmentForOperator(rop);
+
+            for (Mutable<ILogicalOperator> parentRef : originalCandidateParents) {
+                AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getValue();
+                int index = parent.getInputs().indexOf(candidate);
+                if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+                    parent.getInputs().set(index, ropRef);
+                    rop.getOutputs().add(parentRef);
+                } else {
+                    AbstractLogicalOperator exchange = new ExchangeOperator();
+                    exchange.setPhysicalOperator(new OneToOneExchangePOperator());
+                    MutableObject<ILogicalOperator> exchangeRef = new MutableObject<ILogicalOperator>(exchange);
+                    exchange.getInputs().add(ropRef);
+                    rop.getOutputs().add(exchangeRef);
+                    context.computeAndSetTypeEnvironmentForOperator(exchange);
+                    parent.getInputs().set(index, exchangeRef);
+                    context.computeAndSetTypeEnvironmentForOperator(parent);
+                }
+            }
+            List<LogicalVariable> liveVarsNew = new ArrayList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(candidate.getValue(), liveVarsNew);
+            ArrayList<Mutable<ILogicalExpression>> assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
+            for (LogicalVariable liveVar : liveVarsNew)
+                assignExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(liveVar)));
+            for (Mutable<ILogicalOperator> ref : group) {
+                if (ref.equals(candidate))
+                    continue;
+                ArrayList<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+                Map<LogicalVariable, LogicalVariable> variableMappingBack = new HashMap<LogicalVariable, LogicalVariable>();
+                IsomorphismUtilities.mapVariablesTopDown(ref.getValue(), candidate.getValue(), variableMappingBack);
+                for (int i = 0; i < liveVarsNew.size(); i++) {
+                    liveVars.add(variableMappingBack.get(liveVarsNew.get(i)));
+                }
+
+                AbstractLogicalOperator assignOperator = new AssignOperator(liveVars, assignExprs);
+                assignOperator.setPhysicalOperator(new AssignPOperator());
+                AbstractLogicalOperator projectOperator = new ProjectOperator(liveVars);
+                projectOperator.setPhysicalOperator(new StreamProjectPOperator());
+                AbstractLogicalOperator exchOp = new ExchangeOperator();
+                exchOp.setPhysicalOperator(new OneToOneExchangePOperator());
+                exchOp.getInputs().add(ropRef);
+                MutableObject<ILogicalOperator> exchOpRef = new MutableObject<ILogicalOperator>(exchOp);
+                rop.getOutputs().add(exchOpRef);
+                assignOperator.getInputs().add(exchOpRef);
+                projectOperator.getInputs().add(new MutableObject<ILogicalOperator>(assignOperator));
+
+                // set the types
+                context.computeAndSetTypeEnvironmentForOperator(exchOp);
+                context.computeAndSetTypeEnvironmentForOperator(assignOperator);
+                context.computeAndSetTypeEnvironmentForOperator(projectOperator);
+
+                List<Mutable<ILogicalOperator>> parentOpList = childrenToParents.get(ref);
+                for (Mutable<ILogicalOperator> parentOpRef : parentOpList) {
+                    AbstractLogicalOperator parentOp = (AbstractLogicalOperator) parentOpRef.getValue();
+                    int index = parentOp.getInputs().indexOf(ref);
+                    if (parentOp.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+                        AbstractLogicalOperator parentOpNext = (AbstractLogicalOperator) childrenToParents
+                                .get(parentOpRef).get(0).getValue();
+                        if (parentOpNext.isMap()) {
+                            index = parentOpNext.getInputs().indexOf(parentOpRef);
+                            parentOp = parentOpNext;
+                        }
+                    }
+
+                    ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator
+                            : projectOperator;
+                    if (parentOp.isMap()) {
+                        parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(childOp));
+                    } else {
+                        AbstractLogicalOperator exchg = new ExchangeOperator();
+                        exchg.setPhysicalOperator(new OneToOneExchangePOperator());
+                        exchg.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
+                        parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(exchg));
+                        context.computeAndSetTypeEnvironmentForOperator(exchg);
+                    }
+                    context.computeAndSetTypeEnvironmentForOperator(parentOp);
+                }
+            }
+            rewritten = true;
+        }
+        return rewritten;
+    }
+
+    private void genCandidates(IOptimizationContext context) throws AlgebricksException {
+        List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+        while (equivalenceClasses.size() > 0) {
+            previousEquivalenceClasses.clear();
+            for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+                List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
+                candidatesCopy.addAll(candidates);
+                previousEquivalenceClasses.add(candidatesCopy);
+            }
+            List<Mutable<ILogicalOperator>> currentLevelOpRefs = new ArrayList<Mutable<ILogicalOperator>>();
+            for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+                if (candidates.size() > 0) {
+                    for (Mutable<ILogicalOperator> opRef : candidates) {
+                        List<Mutable<ILogicalOperator>> refs = childrenToParents.get(opRef);
+                        if (refs != null)
+                            currentLevelOpRefs.addAll(refs);
+                    }
+                }
+                if (currentLevelOpRefs.size() == 0)
+                    continue;
+                candidatesGrow(currentLevelOpRefs, candidates);
+            }
+            if (currentLevelOpRefs.size() == 0)
+                break;
+            prune(context);
+        }
+        if (equivalenceClasses.size() < 1 && previousEquivalenceClasses.size() > 0) {
+            equivalenceClasses.addAll(previousEquivalenceClasses);
+            prune(context);
+        }
+    }
+
+    private void topDownMaterialization(List<Mutable<ILogicalOperator>> tops) {
+        List<Mutable<ILogicalOperator>> candidates = new ArrayList<Mutable<ILogicalOperator>>();
+        List<Mutable<ILogicalOperator>> nextLevel = new ArrayList<Mutable<ILogicalOperator>>();
+        for (Mutable<ILogicalOperator> op : tops) {
+            for (Mutable<ILogicalOperator> opRef : op.getValue().getInputs()) {
+                List<Mutable<ILogicalOperator>> opRefList = childrenToParents.get(opRef);
+                if (opRefList == null) {
+                    opRefList = new ArrayList<Mutable<ILogicalOperator>>();
+                    childrenToParents.put(opRef, opRefList);
+                    nextLevel.add(opRef);
+                }
+                opRefList.add(op);
+            }
+            if (op.getValue().getInputs().size() == 0)
+                candidates.add(op);
+        }
+        if (equivalenceClasses.size() > 0) {
+            equivalenceClasses.get(0).addAll(candidates);
+        } else {
+            equivalenceClasses.add(candidates);
+        }
+        if (nextLevel.size() > 0) {
+            topDownMaterialization(nextLevel);
+        }
+    }
+
+    private void candidatesGrow(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) {
+        List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<Mutable<ILogicalOperator>>();
+        previousCandidates.addAll(candidates);
+        candidates.clear();
+        boolean validCandidate = false;
+        for (Mutable<ILogicalOperator> op : opList) {
+            List<Mutable<ILogicalOperator>> inputs = op.getValue().getInputs();
+            for (int i = 0; i < inputs.size(); i++) {
+                Mutable<ILogicalOperator> inputRef = inputs.get(i);
+                validCandidate = false;
+                for (Mutable<ILogicalOperator> candidate : previousCandidates) {
+                    // if current input is in candidates
+                    if (inputRef.getValue().equals(candidate.getValue())) {
+                        if (inputs.size() == 1) {
+                            validCandidate = true;
+                        } else {
+                            BitSet candidateInputBitMap = opToCandidateInputs.get(op);
+                            if (candidateInputBitMap == null) {
+                                candidateInputBitMap = new BitSet(inputs.size());
+                                opToCandidateInputs.put(op, candidateInputBitMap);
+                            }
+                            candidateInputBitMap.set(i);
+                            if (candidateInputBitMap.cardinality() == inputs.size()) {
+                                validCandidate = true;
+                            }
+                        }
+                        break;
+                    }
+                }
+            }
+            if (!validCandidate)
+                continue;
+            if (!candidates.contains(op))
+                candidates.add(op);
+        }
+    }
+
+    private void prune(IOptimizationContext context) throws AlgebricksException {
+        List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+            List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
+            candidatesCopy.addAll(candidates);
+            previousEquivalenceClasses.add(candidatesCopy);
+        }
+        equivalenceClasses.clear();
+        for (List<Mutable<ILogicalOperator>> candidates : previousEquivalenceClasses) {
+            boolean[] reserved = new boolean[candidates.size()];
+            for (int i = 0; i < reserved.length; i++)
+                reserved[i] = false;
+            for (int i = candidates.size() - 1; i >= 0; i--) {
+                if (reserved[i] == false) {
+                    List<Mutable<ILogicalOperator>> equivalentClass = new ArrayList<Mutable<ILogicalOperator>>();
+                    ILogicalOperator candidate = candidates.get(i).getValue();
+                    equivalentClass.add(candidates.get(i));
+                    for (int j = i - 1; j >= 0; j--) {
+                        ILogicalOperator peer = candidates.get(j).getValue();
+                        if (IsomorphismUtilities.isOperatorIsomorphic(candidate, peer)) {
+                            reserved[i] = true;
+                            reserved[j] = true;
+                            equivalentClass.add(candidates.get(j));
+                        }
+                    }
+                    if (equivalentClass.size() > 1) {
+                        equivalenceClasses.add(equivalentClass);
+                        Collections.reverse(equivalentClass);
+                    }
+                }
+            }
+            for (int i = candidates.size() - 1; i >= 0; i--) {
+                if (!reserved[i]) {
+                    candidates.remove(i);
+                }
+            }
+        }
+    }
+
+    private boolean[] computeMaterilizationFlags(List<Mutable<ILogicalOperator>> group) {
+        lastUsedClusterId = 0;
+        for (Mutable<ILogicalOperator> root : roots) {
+            computeClusters(null, root, new MutableInt(++lastUsedClusterId));
+        }
+        boolean[] materializationFlags = new boolean[group.size()];
+        boolean worthMaterialization = worthMaterialization(group.get(0));
+        boolean requiresMaterialization;
+        // get clusterIds for each candidate in the group
+        List<Integer> groupClusterIds = new ArrayList<Integer>(group.size());
+        for (int i = 0; i < group.size(); i++) {
+            groupClusterIds.add(clusterMap.get(group.get(i)).getValue());
+        }
+        for (int i = group.size() - 1; i >= 0; i--) {
+            requiresMaterialization = requiresMaterialization(groupClusterIds, i);
+            if (requiresMaterialization && !worthMaterialization) {
+                group.remove(i);
+                groupClusterIds.remove(i);
+            }
+            materializationFlags[i] = requiresMaterialization;
+        }
+        if (group.size() < 2) {
+            group.clear();
+        }
+        // if does not worth materialization, the flags for the remaining candidates should be false
+        return worthMaterialization ? materializationFlags : new boolean[group.size()];
+    }
+
+    private boolean requiresMaterialization(List<Integer> groupClusterIds, int index) {
+        Integer clusterId = groupClusterIds.get(index);
+        BitSet blockingClusters = new BitSet();
+        getAllBlockingClusterIds(clusterId, blockingClusters);
+        if (!blockingClusters.isEmpty()) {
+            for (int i = 0; i < groupClusterIds.size(); i++) {
+                if (i == index) {
+                    continue;
+                }
+                if (blockingClusters.get(groupClusterIds.get(i))) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    private void getAllBlockingClusterIds(int clusterId, BitSet blockingClusters) {
+        BitSet waitFor = clusterWaitForMap.get(clusterId);
+        if (waitFor != null) {
+            for (int i = waitFor.nextSetBit(0); i >= 0; i = waitFor.nextSetBit(i + 1)) {
+                getAllBlockingClusterIds(i, blockingClusters);
+            }
+            blockingClusters.or(waitFor);
+        }
+    }
+
+    private void computeClusters(Mutable<ILogicalOperator> parentRef, Mutable<ILogicalOperator> opRef,
+            MutableInt currentClusterId) {
+        // only replicate operator has multiple outputs
+        int outputIndex = 0;
+        if (opRef.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE) {
+            ReplicateOperator rop = (ReplicateOperator) opRef.getValue();
+            List<Mutable<ILogicalOperator>> outputs = rop.getOutputs();
+            for (outputIndex = 0; outputIndex < outputs.size(); outputIndex++) {
+                if (outputs.get(outputIndex).equals(parentRef)) {
+                    break;
+                }
+            }
+        }
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
+        Pair<int[], int[]> labels = aop.getPhysicalOperator().getInputOutputDependencyLabels(opRef.getValue());
+        List<Mutable<ILogicalOperator>> inputs = opRef.getValue().getInputs();
+        for (int i = 0; i < inputs.size(); i++) {
+            Mutable<ILogicalOperator> inputRef = inputs.get(i);
+            if (labels.second[outputIndex] == 1 && labels.first[i] == 0) { // 1 -> 0
+                if (labels.second.length == 1) {
+                    clusterMap.put(opRef, currentClusterId);
+                    // start a new cluster
+                    MutableInt newClusterId = new MutableInt(++lastUsedClusterId);
+                    computeClusters(opRef, inputRef, newClusterId);
+                    BitSet waitForList = clusterWaitForMap.get(currentClusterId.getValue());
+                    if (waitForList == null) {
+                        waitForList = new BitSet();
+                        clusterWaitForMap.put(currentClusterId.getValue(), waitForList);
+                    }
+                    waitForList.set(newClusterId.getValue());
+                }
+            } else { // 0 -> 0 and 1 -> 1
+                MutableInt prevClusterId = clusterMap.get(opRef);
+                if (prevClusterId == null || prevClusterId.getValue().equals(currentClusterId.getValue())) {
+                    clusterMap.put(opRef, currentClusterId);
+                    computeClusters(opRef, inputRef, currentClusterId);
+                } else {
+                    // merge prevClusterId and currentClusterId: update all the map entries that has currentClusterId to prevClusterId
+                    for (BitSet bs : clusterWaitForMap.values()) {
+                        if (bs.get(currentClusterId.getValue())) {
+                            bs.clear(currentClusterId.getValue());
+                            bs.set(prevClusterId.getValue());
+                        }
+                    }
+                    currentClusterId.setValue(prevClusterId.getValue());
+                }
+            }
+        }
+    }
+
+    protected boolean worthMaterialization(Mutable<ILogicalOperator> candidate) {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) candidate.getValue();
+        if (aop.getPhysicalOperator().expensiveThanMaterialization()) {
+            return true;
+        }
+        List<Mutable<ILogicalOperator>> inputs = candidate.getValue().getInputs();
+        for (Mutable<ILogicalOperator> inputRef : inputs) {
+            if (worthMaterialization(inputRef)) {
+                return true;
+            }
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java
new file mode 100644
index 0000000..8f8e62b
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractFunctionsFromJoinConditionRule.java
@@ -0,0 +1,155 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Factors out function expressions from each comparison function or similarity function in join condition by
+ * assigning them to a variables, and replacing the function expressions with references to those variables.
+ * Examples:
+ * Plan with function expressions in comparison or similarity condition of join expression.
+ * Generates one assign operator per extracted function expression.
+ *
+ * <pre>
+ * Before plan:
+ * 
+ *   join ( eq( funcX($$1), funcX($$2) ) )
+ * 
+ * After plan:
+ * 
+ *   join (eq($$3,$$4))
+ *   assign [$$4] <- [funcY($$2)]
+ *   assign [$$3] <- [funcX($$1)]
+ * </pre>
+ */
+public class ExtractFunctionsFromJoinConditionRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+        if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+                && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
+        ILogicalExpression expr = joinOp.getCondition().getValue();
+
+        return assignFunctionExpressions(joinOp, expr, context);
+
+    }
+
+    private boolean assignFunctionExpressions(AbstractLogicalOperator joinOp, ILogicalExpression expr,
+            IOptimizationContext context) throws AlgebricksException {
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
+        FunctionIdentifier fi = fexp.getFunctionIdentifier();
+
+        boolean modified = false;
+        if (fi.equals(AlgebricksBuiltinFunctions.AND) || fi.equals(AlgebricksBuiltinFunctions.OR)
+                || processArgumentsToFunction(fi)) {
+            for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
+                if (assignFunctionExpressions(joinOp, a.getValue(), context)) {
+                    modified = true;
+                }
+            }
+            return modified;
+        } else if (AlgebricksBuiltinFunctions.isComparisonFunction(fi) || isComparisonFunction(fi)) {
+            for (Mutable<ILogicalExpression> exprRef : fexp.getArguments()) {
+                if (exprRef.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                    LogicalVariable newVar = context.newVar();
+                    AssignOperator newAssign = new AssignOperator(newVar, new MutableObject<ILogicalExpression>(exprRef
+                            .getValue().cloneExpression()));
+                    newAssign.setExecutionMode(joinOp.getExecutionMode());
+
+                    // Place assign below joinOp.
+                    List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getUsedVariables(newAssign, used);
+
+                    Mutable<ILogicalOperator> leftBranchRef = joinOp.getInputs().get(0);
+                    ILogicalOperator leftBranch = leftBranchRef.getValue();
+                    List<LogicalVariable> leftBranchVariables = new ArrayList<LogicalVariable>();
+                    VariableUtilities.getLiveVariables(leftBranch, leftBranchVariables);
+                    if (leftBranchVariables.containsAll(used)) {
+                        // place assign on left branch
+                        newAssign.getInputs().add(new MutableObject<ILogicalOperator>(leftBranch));
+                        leftBranchRef.setValue(newAssign);
+                        modified = true;
+                    } else {
+                        Mutable<ILogicalOperator> rightBranchRef = joinOp.getInputs().get(1);
+                        ILogicalOperator rightBranch = rightBranchRef.getValue();
+                        List<LogicalVariable> rightBranchVariables = new ArrayList<LogicalVariable>();
+                        VariableUtilities.getLiveVariables(rightBranch, rightBranchVariables);
+                        if (rightBranchVariables.containsAll(used)) {
+                            // place assign on right branch
+                            newAssign.getInputs().add(new MutableObject<ILogicalOperator>(rightBranch));
+                            rightBranchRef.setValue(newAssign);
+                            modified = true;
+                        }
+                    }
+
+                    if (modified) {
+                        // Replace original expr with variable reference.
+                        exprRef.setValue(new VariableReferenceExpression(newVar));
+                        context.computeAndSetTypeEnvironmentForOperator(newAssign);
+                        context.computeAndSetTypeEnvironmentForOperator(joinOp);
+                    }
+                }
+            }
+            return modified;
+        } else {
+            return false;
+        }
+    }
+
+    protected boolean processArgumentsToFunction(FunctionIdentifier fi) {
+        return false;
+    }
+
+    protected boolean isComparisonFunction(FunctionIdentifier fi) {
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
new file mode 100644
index 0000000..2790166
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+
+/**
+ * Needed only bc. current Hyrax operators require keys to be fields.
+ */
+public class ExtractGbyExpressionsRule extends AbstractExtractExprRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+
+        if (context.checkIfInDontApplySet(this, op1)) {
+            return false;
+        }
+        context.addToDontApplySet(this, op1);
+        GroupByOperator g = (GroupByOperator) op1;
+        boolean r1 = gbyExprWasRewritten(g, context);
+        boolean r2 = decorExprWasRewritten(g, context);
+        boolean fired = r1 || r2;
+        if (fired) {
+            context.computeAndSetTypeEnvironmentForOperator(g);
+        }
+        return fired;
+    }
+
+    private boolean gbyExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
+        if (!gbyHasComplexExpr(g)) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
+            ILogicalExpression expr = gbyPair.second.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
+                gbyPair.second.setValue(new VariableReferenceExpression(v));
+            }
+        }
+        return true;
+    }
+
+    private boolean decorExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
+        if (!decorHasComplexExpr(g)) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : g.getDecorList()) {
+            ILogicalExpression expr = decorPair.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
+                decorPair.second.setValue(new VariableReferenceExpression(v));
+            }
+        }
+        return true;
+    }
+
+    private boolean gbyHasComplexExpr(GroupByOperator g) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
+            if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean decorHasComplexExpr(GroupByOperator g) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getDecorList()) {
+            if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
new file mode 100644
index 0000000..6cb01f2
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class FactorRedundantGroupAndDecorVarsRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        GroupByOperator gby = (GroupByOperator) op;
+        Map<LogicalVariable, LogicalVariable> varRhsToLhs = new HashMap<LogicalVariable, LogicalVariable>();
+        boolean gvChanged = factorRedundantRhsVars(gby.getGroupByList(), opRef, varRhsToLhs, context);
+        boolean dvChanged = factorRedundantRhsVars(gby.getDecorList(), opRef, varRhsToLhs, context);
+
+        return gvChanged || dvChanged;
+    }
+
+    private boolean factorRedundantRhsVars(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> veList,
+            Mutable<ILogicalOperator> opRef, Map<LogicalVariable, LogicalVariable> varRhsToLhs,
+            IOptimizationContext context) throws AlgebricksException {
+        varRhsToLhs.clear();
+        ListIterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = veList.listIterator();
+        boolean changed = false;
+        while (iter.hasNext()) {
+            Pair<LogicalVariable, Mutable<ILogicalExpression>> p = iter.next();
+            if (p.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                continue;
+            }
+            LogicalVariable v = GroupByOperator.getDecorVariable(p);
+            LogicalVariable lhs = varRhsToLhs.get(v);
+            if (lhs != null) {
+                if (p.first != null) {
+                    AssignOperator assign = new AssignOperator(p.first, new MutableObject<ILogicalExpression>(
+                            new VariableReferenceExpression(lhs)));
+                    ILogicalOperator op = opRef.getValue();
+                    assign.getInputs().add(new MutableObject<ILogicalOperator>(op));
+                    opRef.setValue(assign);
+                    context.computeAndSetTypeEnvironmentForOperator(assign);
+                }
+                iter.remove();
+                changed = true;
+            } else {
+                varRhsToLhs.put(v, p.first);
+            }
+        }
+        return changed;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InferTypesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InferTypesRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InferTypesRule.java
new file mode 100644
index 0000000..3530a5f
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InferTypesRule.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InferTypesRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
+        if (context.getOutputTypeEnvironment(op) != null) {
+            return false;
+        }
+        context.computeAndSetTypeEnvironmentForOperator(op);
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
new file mode 100644
index 0000000..99a1fae
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineAssignIntoAggregateRule.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.AbstractConstVarFunVisitor;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InlineAssignIntoAggregateRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        boolean changed = false;
+        GroupByOperator gbyOp = (GroupByOperator) op;
+        for (ILogicalPlan p : gbyOp.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                if (inlined(r)) {
+                    changed = true;
+                }
+            }
+        }
+        return changed;
+    }
+
+    private boolean inlined(Mutable<ILogicalOperator> r) throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AggregateOperator agg = (AggregateOperator) op1;
+        AssignOperator assign = (AssignOperator) op2;
+        VarExprSubstitution ves = new VarExprSubstitution(assign.getVariables(), assign.getExpressions());
+        for (Mutable<ILogicalExpression> exprRef : agg.getExpressions()) {
+            ILogicalExpression expr = exprRef.getValue();
+            Pair<Boolean, ILogicalExpression> p = expr.accept(ves, null);
+            if (p.first == true) {
+                exprRef.setValue(p.second);
+            }
+            // AbstractLogicalExpression ale = (AbstractLogicalExpression) expr;
+            // ale.accept(ves, null);
+        }
+        List<Mutable<ILogicalOperator>> op1InpList = op1.getInputs();
+        op1InpList.clear();
+        op1InpList.add(op2.getInputs().get(0));
+        return true;
+    }
+
+    private class VarExprSubstitution extends AbstractConstVarFunVisitor<Pair<Boolean, ILogicalExpression>, Void> {
+
+        private List<LogicalVariable> variables;
+        private List<Mutable<ILogicalExpression>> expressions;
+
+        public VarExprSubstitution(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+            this.variables = variables;
+            this.expressions = expressions;
+        }
+
+        @Override
+        public Pair<Boolean, ILogicalExpression> visitConstantExpression(ConstantExpression expr, Void arg) {
+            return new Pair<Boolean, ILogicalExpression>(false, expr);
+        }
+
+        @Override
+        public Pair<Boolean, ILogicalExpression> visitFunctionCallExpression(AbstractFunctionCallExpression expr,
+                Void arg) throws AlgebricksException {
+            boolean changed = false;
+            for (Mutable<ILogicalExpression> eRef : expr.getArguments()) {
+                ILogicalExpression e = eRef.getValue();
+                Pair<Boolean, ILogicalExpression> p = e.accept(this, arg);
+                if (p.first) {
+                    eRef.setValue(p.second);
+                    changed = true;
+                }
+            }
+            return new Pair<Boolean, ILogicalExpression>(changed, expr);
+        }
+
+        @Override
+        public Pair<Boolean, ILogicalExpression> visitVariableReferenceExpression(VariableReferenceExpression expr,
+                Void arg) {
+            LogicalVariable v = expr.getVariableReference();
+            int idx = variables.indexOf(v);
+            if (idx < 0) {
+                return new Pair<Boolean, ILogicalExpression>(false, expr);
+            } else {
+                return new Pair<Boolean, ILogicalExpression>(true, expressions.get(idx).getValue());
+            }
+
+        }
+
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java
new file mode 100644
index 0000000..9ae94ba
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineSingleReferenceVariablesRule.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+
+/**
+ * Inlines variables that are referenced exactly once.
+ * 
+ * Preconditions/Assumptions:
+ * Assumes no projects are in the plan.
+ * 
+ * Example assuming variable $$3 is referenced exactly once.
+ * 
+ * Before plan:
+ * select (funcA($$3))
+ *   ...
+ *     assign [$$3] <- [field-access($$0, 1)]
+ * 
+ * After plan:
+ * select (funcA(field-access($$0, 1))
+ *   ...
+ *     assign [] <- []
+ */
+public class InlineSingleReferenceVariablesRule extends InlineVariablesRule {
+
+    // Maps from variable to a list of operators using that variable.
+    protected Map<LogicalVariable, List<ILogicalOperator>> usedVarsMap = new HashMap<LogicalVariable, List<ILogicalOperator>>();
+    protected List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+    
+    @Override
+    protected void prepare(IOptimizationContext context) {
+        super.prepare(context);
+        usedVarsMap.clear();
+        usedVars.clear();
+    }
+    
+    @Override
+    protected boolean performFinalAction() throws AlgebricksException {
+        boolean modified = false;
+        for (Map.Entry<LogicalVariable, List<ILogicalOperator>> entry : usedVarsMap.entrySet()) {
+            // Perform replacement only if variable is referenced a single time.
+            if (entry.getValue().size() == 1) {
+                ILogicalOperator op = entry.getValue().get(0);
+                if (!op.requiresVariableReferenceExpressions()) {
+                    inlineVisitor.setOperator(op);
+                    inlineVisitor.setTargetVariable(entry.getKey());
+                    if (op.acceptExpressionTransform(inlineVisitor)) {
+                        modified = true;
+                    }
+                    inlineVisitor.setTargetVariable(null);
+                }         
+            }
+        }
+        return modified;
+    }
+
+    @Override
+    protected boolean performBottomUpAction(AbstractLogicalOperator op) throws AlgebricksException {
+        usedVars.clear();
+        VariableUtilities.getUsedVariables(op, usedVars);
+        for (LogicalVariable var : usedVars) {
+            List<ILogicalOperator> opsUsingVar = usedVarsMap.get(var);
+            if (opsUsingVar == null) {
+                opsUsingVar = new ArrayList<ILogicalOperator>();
+                usedVarsMap.put(var, opsUsingVar);
+            }
+            opsUsingVar.add(op);
+        }
+        return false;
+    }
+}

[11/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java
deleted file mode 100644
index 1862834..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantGroupByDecorVars.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Removes duplicate variables from a group-by operator's decor list.
- */
-public class RemoveRedundantGroupByDecorVars implements IAlgebraicRewriteRule {
-
-    private final Set<LogicalVariable> vars = new HashSet<LogicalVariable>();
-    
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        if (context.checkIfInDontApplySet(this, op)) {
-            return false;
-        }
-        vars.clear();
-        
-        boolean modified = false;
-        GroupByOperator groupOp = (GroupByOperator) op;
-        Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
-        while (iter.hasNext()) {
-            Pair<LogicalVariable, Mutable<ILogicalExpression>> decor = iter.next();
-            if (decor.first != null || decor.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                continue;
-            }
-            VariableReferenceExpression varRefExpr = (VariableReferenceExpression) decor.second.getValue();
-            LogicalVariable var = varRefExpr.getVariableReference();
-            if (vars.contains(var)) {
-                iter.remove();
-                modified = true;
-            } else {
-                vars.add(var);
-            }
-        }
-        if (modified) {
-            context.addToDontApplySet(this, op);
-        }
-        return modified;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
deleted file mode 100644
index 10091b8..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/*
- *  project [var-list1]
- *   project [var-list2]
- *     P
- * 
- *  if var-list1.equals(var-list2) becomes
- * 
- *  project [var-list1]
- *    P
- *  
- */
-
-public class RemoveRedundantProjectionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-            Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-            if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
-                return false;
-            }
-            ProjectOperator pi2 = (ProjectOperator) op2;
-            opRef2.setValue(pi2.getInputs().get(0).getValue());
-        } else {
-            if (op1.getInputs().size() <= 0)
-                return false;
-            Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-            if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
-                return false;
-            }
-            if (op2.getInputs().size() <= 0)
-                return false;
-            Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
-            AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
-
-            List<LogicalVariable> liveVars2 = new ArrayList<LogicalVariable>();
-            List<LogicalVariable> liveVars3 = new ArrayList<LogicalVariable>();
-
-            VariableUtilities.getLiveVariables(op2, liveVars2);
-            VariableUtilities.getLiveVariables(op3, liveVars3);
-
-            if (!VariableUtilities.varListEqualUnordered(liveVars2, liveVars3))
-                return false;
-            opRef2.setValue(op3);
-        }
-
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
deleted file mode 100644
index 8b1aeb4..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantVariablesRule.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Replaces redundant variable references with their bottom-most equivalent representative.
- * Does a DFS sweep over the plan keeping track of variable equivalence classes.
- * For example, this rule would perform the following rewrite.
- * Before Plan:
- * select (function-call: func, Args:[%0->$$11])
- * project [$11]
- * assign [$$11] <- [$$10]
- * assign [$$10] <- [$$9]
- * assign [$$9] <- ...
- * ...
- * After Plan:
- * select (function-call: func, Args:[%0->$$9])
- * project [$9]
- * assign [$$11] <- [$$9]
- * assign [$$10] <- [$$9]
- * assign [$$9] <- ...
- * ...
- */
-public class RemoveRedundantVariablesRule implements IAlgebraicRewriteRule {
-
-    private final VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor();
-    private final Map<LogicalVariable, List<LogicalVariable>> equivalentVarsMap = new HashMap<LogicalVariable, List<LogicalVariable>>();
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
-            return false;
-        }
-        boolean modified = removeRedundantVariables(opRef, context);
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
-        }
-        return modified;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    private void updateEquivalenceClassMap(LogicalVariable lhs, LogicalVariable rhs) {
-        List<LogicalVariable> equivalentVars = equivalentVarsMap.get(rhs);
-        if (equivalentVars == null) {
-            equivalentVars = new ArrayList<LogicalVariable>();
-            // The first element in the list is the bottom-most representative which will replace all equivalent vars.
-            equivalentVars.add(rhs);
-            equivalentVars.add(lhs);
-            equivalentVarsMap.put(rhs, equivalentVars);
-        }
-        equivalentVarsMap.put(lhs, equivalentVars);
-        equivalentVars.get(0);
-    }
-
-    private boolean removeRedundantVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        boolean modified = false;
-
-        // Update equivalence class map.
-        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-            AssignOperator assignOp = (AssignOperator) op;
-            int numVars = assignOp.getVariables().size();
-            for (int i = 0; i < numVars; i++) {
-                ILogicalExpression expr = assignOp.getExpressions().get(i).getValue();
-                if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                    continue;
-                }
-                VariableReferenceExpression rhsVarRefExpr = (VariableReferenceExpression) expr;
-                // Update equivalence class map.
-                LogicalVariable lhs = assignOp.getVariables().get(i);
-                LogicalVariable rhs = rhsVarRefExpr.getVariableReference();
-                updateEquivalenceClassMap(lhs, rhs);
-            }
-        }
-
-        // Replace variable references with their first representative. 
-        if (op.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-            // The project operator does not use expressions, so we need to replace it's variables manually.
-            if (replaceProjectVars((ProjectOperator) op)) {
-                modified = true;
-            }
-        } else if (op.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
-            // Replace redundant variables manually in the UnionAll operator.
-            if (replaceUnionAllVars((UnionAllOperator) op)) {
-                modified = true;
-            }
-        } else {
-            if (op.acceptExpressionTransform(substVisitor)) {
-                modified = true;
-            }
-        }
-
-        // Perform variable replacement in nested plans. 
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
-                    if (removeRedundantVariables(rootRef, context)) {
-                        modified = true;
-                    }
-                }
-            }
-        }
-
-        // Deal with re-mapping of variables in group by.
-        if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
-            if (handleGroupByVarRemapping((GroupByOperator) op)) {
-                modified = true;
-            }
-        }
-
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(op);
-            context.addToDontApplySet(this, op);
-        }
-        return modified;
-    }
-
-    private boolean handleGroupByVarRemapping(GroupByOperator groupOp) {
-        boolean modified = false;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gp : groupOp.getGroupByList()) {
-            if (gp.first == null || gp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                continue;
-            }
-            LogicalVariable groupByVar = ((VariableReferenceExpression) gp.second.getValue()).getVariableReference();
-            Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
-            while (iter.hasNext()) {
-                Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
-                if (dp.first != null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                    continue;
-                }
-                LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference();
-                if (dv == groupByVar) {
-                    // The decor variable is redundant, since it is propagated as a grouping variable.
-                    List<LogicalVariable> equivalentVars = equivalentVarsMap.get(groupByVar);
-                    if (equivalentVars != null) {
-                        // Change representative of this equivalence class.
-                        equivalentVars.set(0, gp.first);
-                        equivalentVarsMap.put(gp.first, equivalentVars);
-                    } else {
-                        updateEquivalenceClassMap(gp.first, groupByVar);
-                    }
-                    iter.remove();
-                    modified = true;
-                    break;
-                }
-            }
-        }
-        // find the redundant variables within the decor list
-        Map<LogicalVariable, LogicalVariable> variableToFirstDecorMap = new HashMap<LogicalVariable, LogicalVariable>();
-        Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = groupOp.getDecorList().iterator();
-        while (iter.hasNext()) {
-            Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
-            if (dp.first == null || dp.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                continue;
-            }
-            LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue()).getVariableReference();
-            LogicalVariable firstDecor = variableToFirstDecorMap.get(dv);
-            if (firstDecor == null) {
-                variableToFirstDecorMap.put(dv, dp.first);
-            } else {
-                // The decor variable dp.first is redundant since firstDecor is exactly the same.
-                updateEquivalenceClassMap(dp.first, firstDecor);
-                iter.remove();
-                modified = true;
-            }
-        }
-        return modified;
-    }
-
-    /**
-     * Replace the projects's variables with their corresponding representative
-     * from the equivalence class map (if any).
-     * We cannot use the VariableSubstitutionVisitor here because the project ops
-     * maintain their variables as a list and not as expressions.
-     */
-    private boolean replaceProjectVars(ProjectOperator op) throws AlgebricksException {
-        List<LogicalVariable> vars = op.getVariables();
-        int size = vars.size();
-        boolean modified = false;
-        for (int i = 0; i < size; i++) {
-            LogicalVariable var = vars.get(i);
-            List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
-            if (equivalentVars == null) {
-                continue;
-            }
-            // Replace with equivalence class representative.
-            LogicalVariable representative = equivalentVars.get(0);
-            if (representative != var) {
-                vars.set(i, equivalentVars.get(0));
-                modified = true;
-            }
-        }
-        return modified;
-    }
-
-    private boolean replaceUnionAllVars(UnionAllOperator op) throws AlgebricksException {
-        boolean modified = false;
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : op.getVariableMappings()) {
-            List<LogicalVariable> firstEquivalentVars = equivalentVarsMap.get(varMapping.first);
-            List<LogicalVariable> secondEquivalentVars = equivalentVarsMap.get(varMapping.second);
-            // Replace variables with their representative.
-            if (firstEquivalentVars != null) {
-                varMapping.first = firstEquivalentVars.get(0);
-                modified = true;
-            }
-            if (secondEquivalentVars != null) {
-                varMapping.second = secondEquivalentVars.get(0);
-                modified = true;
-            }
-        }
-        return modified;
-    }
-
-    private class VariableSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
-        @Override
-        public boolean transform(Mutable<ILogicalExpression> exprRef) {
-            ILogicalExpression e = exprRef.getValue();
-            switch (((AbstractLogicalExpression) e).getExpressionTag()) {
-                case VARIABLE: {
-                    // Replace variable references with their equivalent representative in the equivalence class map.
-                    VariableReferenceExpression varRefExpr = (VariableReferenceExpression) e;
-                    LogicalVariable var = varRefExpr.getVariableReference();
-                    List<LogicalVariable> equivalentVars = equivalentVarsMap.get(var);
-                    if (equivalentVars == null) {
-                        return false;
-                    }
-                    LogicalVariable representative = equivalentVars.get(0);
-                    if (representative != var) {
-                        varRefExpr.setVariable(representative);
-                        return true;
-                    }
-                    return false;
-                }
-                case FUNCTION_CALL: {
-                    AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) e;
-                    boolean modified = false;
-                    for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
-                        if (transform(arg)) {
-                            modified = true;
-                        }
-                    }
-                    return modified;
-                }
-                default: {
-                    return false;
-                }
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
deleted file mode 100644
index de4fbfb..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Removes unused variables from Assign, Unnest, Aggregate, and UnionAll operators.
- */
-public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
-            return false;
-        }
-        Set<LogicalVariable> toRemove = new HashSet<LogicalVariable>();
-        collectUnusedAssignedVars((AbstractLogicalOperator) opRef.getValue(), toRemove, true, context);
-        boolean smthToRemove = !toRemove.isEmpty();
-        if (smthToRemove) {
-            removeUnusedAssigns(opRef, toRemove, context);
-        }
-        return !toRemove.isEmpty();
-    }
-
-    private void removeUnusedAssigns(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> toRemove,
-            IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        while (removeFromAssigns(op, toRemove, context) == 0) {
-            if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
-                break;
-            }
-            op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-            opRef.setValue(op);
-        }
-        Iterator<Mutable<ILogicalOperator>> childIter = op.getInputs().iterator();
-        while (childIter.hasNext()) {
-            Mutable<ILogicalOperator> cRef = childIter.next();
-            removeUnusedAssigns(cRef, toRemove, context);
-        }
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans opWithNest = (AbstractOperatorWithNestedPlans) op;
-            Iterator<ILogicalPlan> planIter = opWithNest.getNestedPlans().iterator();
-            while (planIter.hasNext()) {
-                ILogicalPlan p = planIter.next();
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    removeUnusedAssigns(r, toRemove, context);
-                }
-            }
-
-            // Removes redundant nested plans that produces nothing
-            for (int i = opWithNest.getNestedPlans().size() - 1; i >= 0; i--) {
-                ILogicalPlan nestedPlan = opWithNest.getNestedPlans().get(i);
-                List<Mutable<ILogicalOperator>> rootsToBeRemoved = new ArrayList<Mutable<ILogicalOperator>>();
-                for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
-                    ILogicalOperator topOp = r.getValue();
-                    Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
-                    VariableUtilities.getProducedVariablesInDescendantsAndSelf(topOp, producedVars);
-                    if (producedVars.size() == 0) {
-                        rootsToBeRemoved.add(r);
-                    }
-                }
-                // Makes sure the operator should have at least ONE nested plan even it is empty 
-                // (because a lot of places uses this assumption,  TODO(yingyib): clean them up).
-                if (nestedPlan.getRoots().size() == rootsToBeRemoved.size() && opWithNest.getNestedPlans().size() > 1) {
-                    nestedPlan.getRoots().removeAll(rootsToBeRemoved);
-                    opWithNest.getNestedPlans().remove(nestedPlan);
-                }
-            }
-        }
-    }
-
-    private int removeFromAssigns(AbstractLogicalOperator op, Set<LogicalVariable> toRemove,
-            IOptimizationContext context) throws AlgebricksException {
-        switch (op.getOperatorTag()) {
-            case ASSIGN: {
-                AssignOperator assign = (AssignOperator) op;
-                if (removeUnusedVarsAndExprs(toRemove, assign.getVariables(), assign.getExpressions())) {
-                    context.computeAndSetTypeEnvironmentForOperator(assign);
-                }
-                return assign.getVariables().size();
-            }
-            case AGGREGATE: {
-                AggregateOperator agg = (AggregateOperator) op;
-                if (removeUnusedVarsAndExprs(toRemove, agg.getVariables(), agg.getExpressions())) {
-                    context.computeAndSetTypeEnvironmentForOperator(agg);
-                }
-                return agg.getVariables().size();
-            }
-            case UNNEST: {
-                UnnestOperator uOp = (UnnestOperator) op;
-                LogicalVariable pVar = uOp.getPositionalVariable();
-                if (pVar != null && toRemove.contains(pVar)) {
-                    uOp.setPositionalVariable(null);
-                }
-                break;
-            }
-            case UNIONALL: {
-                UnionAllOperator unionOp = (UnionAllOperator) op;
-                if (removeUnusedVarsFromUnionAll(unionOp, toRemove)) {
-                    context.computeAndSetTypeEnvironmentForOperator(unionOp);
-                }
-                return unionOp.getVariableMappings().size();
-            }
-        }
-        return -1;
-    }
-
-    private boolean removeUnusedVarsFromUnionAll(UnionAllOperator unionOp, Set<LogicalVariable> toRemove) {
-        Iterator<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> iter = unionOp.getVariableMappings()
-                .iterator();
-        boolean modified = false;
-        Set<LogicalVariable> removeFromRemoveSet = new HashSet<LogicalVariable>();
-        while (iter.hasNext()) {
-            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping = iter.next();
-            if (toRemove.contains(varMapping.third)) {
-                iter.remove();
-                modified = true;
-            }
-            // In any case, make sure we do not removing these variables.
-            removeFromRemoveSet.add(varMapping.first);
-            removeFromRemoveSet.add(varMapping.second);
-        }
-        toRemove.removeAll(removeFromRemoveSet);
-        return modified;
-    }
-
-    private boolean removeUnusedVarsAndExprs(Set<LogicalVariable> toRemove, List<LogicalVariable> varList,
-            List<Mutable<ILogicalExpression>> exprList) {
-        boolean changed = false;
-        Iterator<LogicalVariable> varIter = varList.iterator();
-        Iterator<Mutable<ILogicalExpression>> exprIter = exprList.iterator();
-        while (varIter.hasNext()) {
-            LogicalVariable v = varIter.next();
-            exprIter.next();
-            if (toRemove.contains(v)) {
-                varIter.remove();
-                exprIter.remove();
-                changed = true;
-            }
-        }
-        return changed;
-    }
-
-    private void collectUnusedAssignedVars(AbstractLogicalOperator op, Set<LogicalVariable> toRemove, boolean first,
-            IOptimizationContext context) throws AlgebricksException {
-        if (!first) {
-            context.addToDontApplySet(this, op);
-        }
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            collectUnusedAssignedVars((AbstractLogicalOperator) c.getValue(), toRemove, false, context);
-        }
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : plan.getRoots()) {
-                    collectUnusedAssignedVars((AbstractLogicalOperator) r.getValue(), toRemove, false, context);
-                }
-            }
-        }
-        boolean removeUsedVars = true;
-        switch (op.getOperatorTag()) {
-            case ASSIGN: {
-                AssignOperator assign = (AssignOperator) op;
-                toRemove.addAll(assign.getVariables());
-                break;
-            }
-            case AGGREGATE: {
-                AggregateOperator agg = (AggregateOperator) op;
-                toRemove.addAll(agg.getVariables());
-                break;
-            }
-            case UNNEST: {
-                UnnestOperator uOp = (UnnestOperator) op;
-                LogicalVariable pVar = uOp.getPositionalVariable();
-                if (pVar != null) {
-                    toRemove.add(pVar);
-                }
-                break;
-            }
-            case UNIONALL: {
-                UnionAllOperator unionOp = (UnionAllOperator) op;
-                for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping : unionOp
-                        .getVariableMappings()) {
-                    toRemove.add(varMapping.third);
-                }
-                removeUsedVars = false;
-                break;
-            }
-        }
-        if (removeUsedVars) {
-            List<LogicalVariable> used = new LinkedList<LogicalVariable>();
-            VariableUtilities.getUsedVariables(op, used);
-            toRemove.removeAll(used);
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
deleted file mode 100644
index bc7255b..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AggregatePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.BulkloadPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.DataSourceScanPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.DistributeResultPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.EmptyTupleSourcePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryStableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.IndexBulkloadPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.IndexInsertDeletePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InsertDeletePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.MicroPreclusteredGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NestedTupleSourcePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreSortedDistinctByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RunningAggregatePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SinkPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SinkWritePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamSelectPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StringStreamingScriptPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SubplanPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.TokenizePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnionAllPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnnestPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.WriteResultPOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.rewriter.util.JoinUtils;
-
-public class SetAlgebricksPhysicalOperatorsRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        // if (context.checkIfInDontApplySet(this, op)) {
-        // return false;
-        // }
-        if (op.getPhysicalOperator() != null) {
-            return false;
-        }
-
-        computeDefaultPhysicalOp(op, true, context);
-        // context.addToDontApplySet(this, op);
-        return true;
-    }
-
-    private static void setPhysicalOperators(ILogicalPlan plan, boolean topLevelOp, IOptimizationContext context)
-            throws AlgebricksException {
-        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
-            computeDefaultPhysicalOp((AbstractLogicalOperator) root.getValue(), topLevelOp, context);
-        }
-    }
-
-    @SuppressWarnings("unchecked")
-    private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, boolean topLevelOp,
-            IOptimizationContext context) throws AlgebricksException {
-        PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
-        if (op.getPhysicalOperator() == null) {
-            switch (op.getOperatorTag()) {
-                case AGGREGATE: {
-                    op.setPhysicalOperator(new AggregatePOperator());
-                    break;
-                }
-                case ASSIGN: {
-                    op.setPhysicalOperator(new AssignPOperator());
-                    break;
-                }
-                case DISTINCT: {
-                    DistinctOperator distinct = (DistinctOperator) op;
-                    distinct.setPhysicalOperator(new PreSortedDistinctByPOperator(distinct.getDistinctByVarList()));
-                    break;
-                }
-                case EMPTYTUPLESOURCE: {
-                    op.setPhysicalOperator(new EmptyTupleSourcePOperator());
-                    break;
-                }
-                case EXCHANGE: {
-                    if (op.getPhysicalOperator() == null) {
-                        throw new AlgebricksException("Implementation for EXCHANGE operator was not set.");
-                    }
-                    // implem. choice for exchange should be set by a parent op.
-                    break;
-                }
-                case GROUP: {
-                    GroupByOperator gby = (GroupByOperator) op;
-
-                    if (gby.getNestedPlans().size() == 1) {
-                        ILogicalPlan p0 = gby.getNestedPlans().get(0);
-                        if (p0.getRoots().size() == 1) {
-                            if (gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE
-                                    || gby.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE) {
-                                if (!topLevelOp) {
-                                    throw new NotImplementedException(
-                                            "External hash group-by for nested grouping is not implemented.");
-                                }
-
-                                boolean hasIntermediateAgg = generateMergeAggregationExpressions(gby, context);
-                                if (hasIntermediateAgg) {
-                                    ExternalGroupByPOperator externalGby = new ExternalGroupByPOperator(
-                                            gby.getGroupByList(),
-                                            physicalOptimizationConfig.getMaxFramesExternalGroupBy(),
-                                            physicalOptimizationConfig.getExternalGroupByTableSize());
-                                    op.setPhysicalOperator(externalGby);
-                                    break;
-                                }
-                            }
-                        }
-                    }
-
-                    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
-                    List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
-                    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
-                        ILogicalExpression expr = p.second.getValue();
-                        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
-                            columnList.add(varRef.getVariableReference());
-                        }
-                    }
-                    if (topLevelOp) {
-                        op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList));
-                    } else {
-                        op.setPhysicalOperator(new MicroPreclusteredGroupByPOperator(columnList));
-                    }
-                    break;
-                }
-                case INNERJOIN: {
-                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, context);
-                    break;
-                }
-                case LEFTOUTERJOIN: {
-                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((LeftOuterJoinOperator) op, context);
-                    break;
-                }
-                case LIMIT: {
-                    op.setPhysicalOperator(new StreamLimitPOperator());
-                    break;
-                }
-                case NESTEDTUPLESOURCE: {
-                    op.setPhysicalOperator(new NestedTupleSourcePOperator());
-                    break;
-                }
-                case ORDER: {
-                    OrderOperator oo = (OrderOperator) op;
-                    for (Pair<IOrder, Mutable<ILogicalExpression>> p : oo.getOrderExpressions()) {
-                        ILogicalExpression e = p.second.getValue();
-                        if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                            throw new AlgebricksException("Order expression " + e + " has not been normalized.");
-                        }
-                    }
-                    if (topLevelOp) {
-                        op.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig
-                                .getMaxFramesExternalSort()));
-                    } else {
-                        op.setPhysicalOperator(new InMemoryStableSortPOperator());
-                    }
-                    break;
-                }
-                case PROJECT: {
-                    op.setPhysicalOperator(new StreamProjectPOperator());
-                    break;
-                }
-                case RUNNINGAGGREGATE: {
-                    op.setPhysicalOperator(new RunningAggregatePOperator());
-                    break;
-                }
-                case REPLICATE: {
-                    op.setPhysicalOperator(new ReplicatePOperator());
-                    break;
-                }
-                case SCRIPT: {
-                    op.setPhysicalOperator(new StringStreamingScriptPOperator());
-                    break;
-                }
-                case SELECT: {
-                    op.setPhysicalOperator(new StreamSelectPOperator());
-                    break;
-                }
-                case SUBPLAN: {
-                    op.setPhysicalOperator(new SubplanPOperator());
-                    break;
-                }
-                case UNIONALL: {
-                    op.setPhysicalOperator(new UnionAllPOperator());
-                    break;
-                }
-
-                case UNNEST: {
-                    op.setPhysicalOperator(new UnnestPOperator());
-                    break;
-                }
-                case DATASOURCESCAN: {
-                    DataSourceScanOperator scan = (DataSourceScanOperator) op;
-                    IDataSource dataSource = scan.getDataSource();
-                    DataSourceScanPOperator dss = new DataSourceScanPOperator(dataSource);
-                    IMetadataProvider mp = context.getMetadataProvider();
-                    if (mp.scannerOperatorIsLeaf(dataSource)) {
-                        dss.disableJobGenBelowMe();
-                    }
-                    op.setPhysicalOperator(dss);
-                    break;
-                }
-                case WRITE: {
-                    op.setPhysicalOperator(new SinkWritePOperator());
-                    break;
-                }
-                case DISTRIBUTE_RESULT: {
-                    op.setPhysicalOperator(new DistributeResultPOperator());
-                    break;
-                }
-                case WRITE_RESULT: {
-                    WriteResultOperator opLoad = (WriteResultOperator) op;
-                    LogicalVariable payload;
-                    List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
-                    List<LogicalVariable> additionalFilteringKeys = null;
-                    payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getKeyExpressions(), keys);
-                    if (opLoad.getAdditionalFilteringExpressions() != null) {
-                        additionalFilteringKeys = new ArrayList<LogicalVariable>();
-                        getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys);
-                    }
-                    op.setPhysicalOperator(new WriteResultPOperator(opLoad.getDataSource(), payload, keys,
-                            additionalFilteringKeys));
-                    break;
-                }
-                case INSERT_DELETE: {
-                    InsertDeleteOperator opLoad = (InsertDeleteOperator) op;
-                    LogicalVariable payload;
-                    List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
-                    List<LogicalVariable> additionalFilteringKeys = null;
-                    payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getPrimaryKeyExpressions(), keys);
-                    if (opLoad.getAdditionalFilteringExpressions() != null) {
-                        additionalFilteringKeys = new ArrayList<LogicalVariable>();
-                        getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys);
-                    }
-                    if (opLoad.isBulkload()) {
-                        op.setPhysicalOperator(new BulkloadPOperator(payload, keys, additionalFilteringKeys, opLoad
-                                .getDataSource()));
-                    } else {
-                        op.setPhysicalOperator(new InsertDeletePOperator(payload, keys, additionalFilteringKeys, opLoad
-                                .getDataSource()));
-                    }
-                    break;
-                }
-                case INDEX_INSERT_DELETE: {
-                    IndexInsertDeleteOperator opInsDel = (IndexInsertDeleteOperator) op;
-                    List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
-                    List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
-                    List<LogicalVariable> additionalFilteringKeys = null;
-                    getKeys(opInsDel.getPrimaryKeyExpressions(), primaryKeys);
-                    getKeys(opInsDel.getSecondaryKeyExpressions(), secondaryKeys);
-                    if (opInsDel.getAdditionalFilteringExpressions() != null) {
-                        additionalFilteringKeys = new ArrayList<LogicalVariable>();
-                        getKeys(opInsDel.getAdditionalFilteringExpressions(), additionalFilteringKeys);
-                    }
-                    if (opInsDel.isBulkload()) {
-                        op.setPhysicalOperator(new IndexBulkloadPOperator(primaryKeys, secondaryKeys,
-                                additionalFilteringKeys, opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex()));
-                    } else {
-                        op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys,
-                                additionalFilteringKeys, opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex()));
-                    }
-
-                    break;
-
-                }
-                case TOKENIZE: {
-                    TokenizeOperator opTokenize = (TokenizeOperator) op;
-                    List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
-                    List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
-                    getKeys(opTokenize.getPrimaryKeyExpressions(), primaryKeys);
-                    getKeys(opTokenize.getSecondaryKeyExpressions(), secondaryKeys);
-                    // Tokenize Operator only operates with a bulk load on a data set with an index
-                    if (opTokenize.isBulkload()) {
-                        op.setPhysicalOperator(new TokenizePOperator(primaryKeys, secondaryKeys, opTokenize
-                                .getDataSourceIndex()));
-                    }
-                    break;
-                }
-                case SINK: {
-                    op.setPhysicalOperator(new SinkPOperator());
-                    break;
-                }
-            }
-        }
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : nested.getNestedPlans()) {
-                setPhysicalOperators(p, false, context);
-            }
-        }
-        for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
-            computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), topLevelOp, context);
-        }
-    }
-
-    private static void getKeys(List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
-        for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
-            ILogicalExpression e = kExpr.getValue();
-            if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new NotImplementedException();
-            }
-            keys.add(((VariableReferenceExpression) e).getVariableReference());
-        }
-    }
-
-    private static LogicalVariable getKeysAndLoad(Mutable<ILogicalExpression> payloadExpr,
-            List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
-        LogicalVariable payload;
-        if (payloadExpr.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-            throw new NotImplementedException();
-        }
-        payload = ((VariableReferenceExpression) payloadExpr.getValue()).getVariableReference();
-
-        for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
-            ILogicalExpression e = kExpr.getValue();
-            if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new NotImplementedException();
-            }
-            keys.add(((VariableReferenceExpression) e).getVariableReference());
-        }
-        return payload;
-    }
-
-    private static boolean generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context)
-            throws AlgebricksException {
-        if (gby.getNestedPlans().size() != 1) {
-            //External/Sort group-by currently works only for one nested plan with one root containing
-            //an aggregate and a nested-tuple-source.
-            throw new AlgebricksException(
-                    "External group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        ILogicalPlan p0 = gby.getNestedPlans().get(0);
-        if (p0.getRoots().size() != 1) {
-            //External/Sort group-by currently works only for one nested plan with one root containing
-            //an aggregate and a nested-tuple-source.
-            throw new AlgebricksException(
-                    "External group-by currently works only for one nested plan with one root containing"
-                            + "an aggregate and a nested-tuple-source.");
-        }
-        IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
-                .getMergeAggregationExpressionFactory();
-        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
-        AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
-        if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
-        List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
-        List<LogicalVariable> originalAggVars = aggOp.getVariables();
-        int n = aggOp.getExpressions().size();
-        List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
-        for (int i = 0; i < n; i++) {
-            ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(
-                    originalAggVars.get(i), aggFuncRefs.get(i).getValue(), context);
-            if (mergeExpr == null) {
-                return false;
-            }
-            mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
-        }
-        aggOp.setMergeExpressions(mergeExpressionRefs);
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
deleted file mode 100644
index d938f73..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This rule sets the executionMode property of an operator, w/o introducing
- * EXCHANGE operators in the plan. Previously, i.e. before having physical
- * optimizations in place, we were using the IntroduceExchangeRule, which was
- * doing both, to both set excutionMode and introduce data exchange ops.
- * 
- * @author Nicola
- */
-public class SetExecutionModeRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        boolean changed = OperatorManipulationUtil.setOperatorMode(op);
-        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED
-                || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) {
-            return changed;
-        }
-        switch (op.getOperatorTag()) {
-        // case DISTINCT:
-        // case AGGREGATE:
-        // case GROUP:
-        // case ORDER:
-        // case INNERJOIN:
-        // case LEFTOUTERJOIN: {
-        // op.setExecutionMode(ExecutionMode.GLOBAL);
-        // return true;
-        // }
-
-            case PARTITIONINGSPLIT: {
-                throw new NotImplementedException();
-            }
-            default: {
-                return changed;
-            }
-        }
-
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
deleted file mode 100644
index f6c6ad8..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class SimpleUnnestToProductRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN
-                && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
-            return false;
-        }
-
-        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-
-        if (!(op2 instanceof AbstractScanOperator) && !descOrSelfIsSourceScan(op2)) {
-            return false;
-        }
-        // Make sure that op does not use any variables produced by op2.
-        if (!opsAreIndependent(op, op2)) {
-            return false;
-        }
-
-        /**
-         * finding the boundary between left branch and right branch
-         * operator pipeline on-top-of boundaryOpRef (exclusive) is the inner branch
-         * operator pipeline under boundaryOpRef (inclusive) is the outer branch
-         */
-        Mutable<ILogicalOperator> currentOpRef = opRef;
-        Mutable<ILogicalOperator> boundaryOpRef = currentOpRef.getValue().getInputs().get(0);
-        while (currentOpRef.getValue().getInputs().size() == 1) {
-            currentOpRef = currentOpRef.getValue().getInputs().get(0);
-        }
-        Mutable<ILogicalOperator> tupleSourceOpRef = currentOpRef;
-        currentOpRef = opRef;
-        if (tupleSourceOpRef.getValue().getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) tupleSourceOpRef.getValue();
-            // If the subplan input is a trivial plan, do not do the rewriting.
-            if (nts.getSourceOperator().getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE) {
-                while (currentOpRef.getValue().getInputs().size() == 1
-                        && currentOpRef.getValue() instanceof AbstractScanOperator
-                        && descOrSelfIsSourceScan((AbstractLogicalOperator) currentOpRef.getValue())) {
-                    if (opsAreIndependent(currentOpRef.getValue(), tupleSourceOpRef.getValue())) {
-                        /** move down the boundary if the operator is independent of the tuple source */
-                        boundaryOpRef = currentOpRef.getValue().getInputs().get(0);
-                    } else {
-                        break;
-                    }
-                    currentOpRef = currentOpRef.getValue().getInputs().get(0);
-                }
-            }
-        }
-
-        /** join the two independent branches */
-        InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE),
-                new MutableObject<ILogicalOperator>(boundaryOpRef.getValue()), new MutableObject<ILogicalOperator>(
-                        opRef.getValue()));
-        opRef.setValue(join);
-        ILogicalOperator ets = new EmptyTupleSourceOperator();
-        boundaryOpRef.setValue(ets);
-        context.computeAndSetTypeEnvironmentForOperator(boundaryOpRef.getValue());
-        context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
-        context.computeAndSetTypeEnvironmentForOperator(join);
-        return true;
-    }
-
-    private boolean descOrSelfIsSourceScan(AbstractLogicalOperator op2) {
-        // Disregard data source scans in a subplan.
-        if (op2.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN
-                && op2.getOperatorTag() != LogicalOperatorTag.UNNEST) {
-            return true;
-        }
-        for (Mutable<ILogicalOperator> cRef : op2.getInputs()) {
-            AbstractLogicalOperator alo = (AbstractLogicalOperator) cRef.getValue();
-            if (descOrSelfIsSourceScan(alo)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private boolean opsAreIndependent(ILogicalOperator unnestOp, ILogicalOperator outer) throws AlgebricksException {
-        if (unnestOp.equals(outer)) {
-            return false;
-        }
-        List<LogicalVariable> opUsedVars = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(unnestOp, opUsedVars);
-        Set<LogicalVariable> op2LiveVars = new HashSet<LogicalVariable>();
-        VariableUtilities.getLiveVariables(outer, op2LiveVars);
-        for (LogicalVariable usedVar : opUsedVars) {
-            if (op2LiveVars.contains(usedVar)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
deleted file mode 100644
index f249122..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Looks for a nested group-by plan ending in
- * subplan {
- * ...
- * }
- * select (function-call: algebricks:not, Args:[function-call:
- * algebricks:is-null, Args:[...]])
- * nested tuple source -- |UNPARTITIONED|
- */
-
-public class SubplanOutOfGroupRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
-        if (op0.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        GroupByOperator gby = (GroupByOperator) op0;
-
-        Iterator<ILogicalPlan> plansIter = gby.getNestedPlans().iterator();
-        ILogicalPlan p = null;
-        while (plansIter.hasNext()) {
-            p = plansIter.next();
-        }
-        if (p == null) {
-            return false;
-        }
-        if (p.getRoots().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> op1Ref = p.getRoots().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getValue();
-        boolean found = false;
-        while (op1.getInputs().size() == 1) {
-            if (op1.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
-                SubplanOperator subplan = (SubplanOperator) op1;
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getValue();
-                if (OperatorPropertiesUtil.isNullTest(op2)) {
-                    if (subplan.getNestedPlans().size() == 1) {
-                        ILogicalPlan p1 = subplan.getNestedPlans().get(0);
-                        if (p1.getRoots().size() == 1) {
-                            AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getValue();
-                            if (r1.getOperatorTag() == LogicalOperatorTag.INNERJOIN
-                                    || r1.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
-                                // now, check that it propagates all variables,
-                                // so it can be pushed
-                                List<LogicalVariable> op2Vars = new ArrayList<LogicalVariable>();
-                                VariableUtilities.getLiveVariables(op2, op2Vars);
-                                List<LogicalVariable> op1Vars = new ArrayList<LogicalVariable>();
-                                VariableUtilities.getLiveVariables(subplan, op1Vars);
-                                if (op1Vars.containsAll(op2Vars)) {
-                                    found = true;
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            op1Ref = op1.getInputs().get(0);
-            op1 = (AbstractLogicalOperator) op1Ref.getValue();
-        }
-        if (!found) {
-            return false;
-        }
-
-        ILogicalOperator subplan = op1;
-        ILogicalOperator op2 = op1.getInputs().get(0).getValue();
-        op1Ref.setValue(op2);
-        Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0);
-        ILogicalOperator opUnder = opUnderRef.getValue();
-        subplan.getInputs().clear();
-        subplan.getInputs().add(new MutableObject<ILogicalOperator>(opUnder));
-        opUnderRef.setValue(subplan);
-
-        return true;
-    }
-}

[16/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
deleted file mode 100644
index 67b2d0c..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceCombinerRule.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public abstract class AbstractIntroduceCombinerRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    /**
-     * Replace the original aggregate functions with their corresponding global aggregate function.
-     */
-    protected void replaceOriginalAggFuncs(Set<SimilarAggregatesInfo> toReplaceSet) {
-        for (SimilarAggregatesInfo sai : toReplaceSet) {
-            for (AggregateExprInfo aei : sai.simAggs) {
-                AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getValue();
-                afce.setFunctionInfo(aei.newFunInfo);
-                afce.getArguments().clear();
-                afce.getArguments().add(new MutableObject<ILogicalExpression>(sai.stepOneResult));
-            }
-        }
-    }
-
-    protected Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg,
-            GroupByOperator newGbyOp, Set<SimilarAggregatesInfo> toReplaceSet, IOptimizationContext context)
-            throws AlgebricksException {
-
-        ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
-        ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
-
-        List<LogicalVariable> initVars = initAgg.getVariables();
-        List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
-        int numExprs = initVars.size();
-
-        // First make sure that all agg funcs are two step, otherwise we cannot use local aggs.
-        for (int i = 0; i < numExprs; i++) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
-            if (!aggFun.isTwoStep()) {
-                return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
-            }
-        }
-
-        boolean haveAggToReplace = false;
-        for (int i = 0; i < numExprs; i++) {
-            Mutable<ILogicalExpression> expRef = initExprs.get(i);
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
-            IFunctionInfo fi1 = aggFun.getStepOneAggregate();
-            // Clone the aggregate's args.
-            List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun
-                    .getArguments().size());
-            for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
-                newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
-            }
-            IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
-
-            SimilarAggregatesInfo inf = new SimilarAggregatesInfo();
-            LogicalVariable newAggVar = context.newVar();
-            pushedVars.add(newAggVar);
-            inf.stepOneResult = new VariableReferenceExpression(newAggVar);
-            inf.simAggs = new ArrayList<AggregateExprInfo>();
-            toReplaceSet.add(inf);
-            AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
-            pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
-            AggregateExprInfo aei = new AggregateExprInfo();
-            aei.aggExprRef = expRef;
-            aei.newFunInfo = fi2;
-            inf.simAggs.add(aei);
-            haveAggToReplace = true;
-        }
-
-        if (!pushedVars.isEmpty()) {
-            AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
-            pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
-            // If newGbyOp is null, then we optimizing an aggregate without group by.
-            if (newGbyOp != null) {
-                // Cut and paste nested input pipelines of initAgg to pushedAgg's input
-                Mutable<ILogicalOperator> inputRef = initAgg.getInputs().get(0);
-                Mutable<ILogicalOperator> bottomRef = inputRef;
-                while (bottomRef.getValue().getInputs().size() > 0) {
-                    bottomRef = bottomRef.getValue().getInputs().get(0);
-                }
-                ILogicalOperator oldNts = bottomRef.getValue();
-                initAgg.getInputs().clear();
-                initAgg.getInputs().add(new MutableObject<ILogicalOperator>(oldNts));
-
-                // Hook up the nested aggregate op with the outer group by.
-                NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(
-                        newGbyOp));
-                nts.setExecutionMode(ExecutionMode.LOCAL);
-                bottomRef.setValue(nts);
-                pushedAgg.getInputs().add(inputRef);
-            } else {
-                // The local aggregate operator is fed by the input of the original aggregate operator.
-                pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(initAgg.getInputs().get(0).getValue()));
-                // Reintroduce assign op for the global agg partitioning var.
-                initAgg.getInputs().get(0).setValue(pushedAgg);
-                pushedAgg.setGlobal(false);
-                context.computeAndSetTypeEnvironmentForOperator(pushedAgg);
-            }
-            return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
-        } else {
-            return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
-        }
-    }
-
-    protected class SimilarAggregatesInfo {
-        ILogicalExpression stepOneResult;
-        List<AggregateExprInfo> simAggs;
-    }
-
-    protected class AggregateExprInfo {
-        Mutable<ILogicalExpression> aggExprRef;
-        IFunctionInfo newFunInfo;
-    }
-
-    protected class BookkeepingInfo {
-        Map<GroupByOperator, List<LogicalVariable>> modifyGbyMap = new HashMap<GroupByOperator, List<LogicalVariable>>();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java
deleted file mode 100644
index b60c80d..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractIntroduceGroupByCombinerRule.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-
-public abstract class AbstractIntroduceGroupByCombinerRule extends AbstractIntroduceCombinerRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (context.checkIfInDontApplySet(this, op)) {
-            return false;
-        }
-        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        GroupByOperator gbyOp = (GroupByOperator) op;
-        if (gbyOp.getExecutionMode() != ExecutionMode.PARTITIONED) {
-            return false;
-        }
-
-        BookkeepingInfo bi = new BookkeepingInfo();
-        GroupByOperator newGbyOp = opToPush(gbyOp, bi, context);
-        if (newGbyOp == null) {
-            return false;
-        }
-
-        Set<LogicalVariable> newGbyLiveVars = new ListSet<LogicalVariable>();
-        VariableUtilities.getLiveVariables(newGbyOp, newGbyLiveVars);
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
-            List<LogicalVariable> usedDecorVars = new ArrayList<LogicalVariable>();
-            // p.second.getValue() should always return a VariableReferenceExpression, hence
-            // usedDecorVars should always contain only one variable.
-            p.second.getValue().getUsedVariables(usedDecorVars);
-            if (!newGbyLiveVars.contains(usedDecorVars.get(0))) {
-                // Let the left-hand side of gbyOp's decoration expressions populated through the combiner group-by without
-                // any intermediate assignment.
-                newGbyOp.addDecorExpression(null, p.second.getValue());
-            }
-        }
-        newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
-        Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
-        newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v);
-
-        Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY);
-        newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2);
-
-        List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(newGbyOp, propagatedVars);
-
-        Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
-        OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars);
-
-        for (LogicalVariable var : freeVars) {
-            if (!propagatedVars.contains(var)) {
-                LogicalVariable newDecorVar = context.newVar();
-                newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
-                VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
-                        newDecorVar, context);
-            }
-        }
-
-        Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
-        opRef3.setValue(newGbyOp);
-        typeGby(newGbyOp, context);
-        typeGby(gbyOp, context);
-        context.addToDontApplySet(this, op);
-        return true;
-    }
-
-    private void typeGby(AbstractOperatorWithNestedPlans op, IOptimizationContext context) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            OperatorPropertiesUtil.typePlan(p, context);
-        }
-        context.computeAndSetTypeEnvironmentForOperator(op);
-    }
-
-    private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context)
-            throws AlgebricksException {
-        // Hook up input to new group-by.
-        Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
-        ILogicalOperator op3 = opRef3.getValue();
-        GroupByOperator newGbyOp = new GroupByOperator();
-        newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
-        // Copy annotations.
-        Map<String, Object> annotations = newGbyOp.getAnnotations();
-        annotations.putAll(gbyOp.getAnnotations());
-
-        List<LogicalVariable> gbyVars = gbyOp.getGbyVarList();
-        for (ILogicalPlan p : gbyOp.getNestedPlans()) {
-            Pair<Boolean, ILogicalPlan> bip = tryToPushSubplan(p, gbyOp, newGbyOp, bi, gbyVars, context);
-            if (!bip.first) {
-                // For now, if we cannot push everything, give up.
-                return null;
-            }
-            ILogicalPlan pushedSubplan = bip.second;
-            if (pushedSubplan != null) {
-                newGbyOp.getNestedPlans().add(pushedSubplan);
-            }
-        }
-
-        ArrayList<LogicalVariable> newOpGbyList = new ArrayList<LogicalVariable>();
-        ArrayList<LogicalVariable> replGbyList = new ArrayList<LogicalVariable>();
-        // Find maximal sequence of variable.
-        for (Map.Entry<GroupByOperator, List<LogicalVariable>> e : bi.modifyGbyMap.entrySet()) {
-            List<LogicalVariable> varList = e.getValue();
-            boolean see1 = true;
-            int sz1 = newOpGbyList.size();
-            int i = 0;
-            for (LogicalVariable v : varList) {
-                if (see1) {
-                    if (i < sz1) {
-                        LogicalVariable v2 = newOpGbyList.get(i);
-                        if (v != v2) {
-                            // cannot linearize
-                            return null;
-                        }
-                    } else {
-                        see1 = false;
-                        newOpGbyList.add(v);
-                        replGbyList.add(context.newVar());
-                    }
-                    i++;
-                } else {
-                    newOpGbyList.add(v);
-                    replGbyList.add(context.newVar());
-                }
-            }
-        }
-        // set the vars in the new op
-        int n = newOpGbyList.size();
-        for (int i = 0; i < n; i++) {
-            newGbyOp.addGbyExpression(replGbyList.get(i), new VariableReferenceExpression(newOpGbyList.get(i)));
-            VariableUtilities.substituteVariables(gbyOp, newOpGbyList.get(i), replGbyList.get(i), false, context);
-        }
-        return newGbyOp;
-    }
-
-    private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan nestedPlan, GroupByOperator oldGbyOp,
-            GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context)
-            throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
-        Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
-        for (Mutable<ILogicalOperator> r : nestedPlan.getRoots()) {
-            if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, pushedRoots, toReplaceSet)) {
-                // For now, if we cannot push everything, give up.
-                return new Pair<Boolean, ILogicalPlan>(false, null);
-            }
-        }
-        if (pushedRoots.isEmpty()) {
-            return new Pair<Boolean, ILogicalPlan>(true, null);
-        } else {
-            // Replaces the aggregation expressions in the original group-by op with new ones.
-            ILogicalPlan newPlan = new ALogicalPlanImpl(pushedRoots);
-            ILogicalPlan plan = fingIdenticalPlan(newGbyOp, newPlan);
-            replaceOriginalAggFuncs(toReplaceSet);
-            if (plan == null) {
-                return new Pair<Boolean, ILogicalPlan>(true, newPlan);
-            } else {
-                // Does not add a nested subplan to newGbyOp if there already exists an isomorphic plan.
-                Set<LogicalVariable> originalVars = new ListSet<LogicalVariable>();
-                Set<LogicalVariable> newVars = new ListSet<LogicalVariable>();
-                for (Mutable<ILogicalOperator> rootRef : pushedRoots) {
-                    VariableUtilities.getProducedVariables(rootRef.getValue(), originalVars);
-                }
-                for (Mutable<ILogicalOperator> rootRef : plan.getRoots()) {
-                    VariableUtilities.getProducedVariables(rootRef.getValue(), newVars);
-                }
-
-                // Replaces variable exprs referring to the variables produced by newPlan by 
-                // those produced by plan.
-                Iterator<LogicalVariable> originalVarIter = originalVars.iterator();
-                Iterator<LogicalVariable> newVarIter = newVars.iterator();
-                while (originalVarIter.hasNext()) {
-                    LogicalVariable originalVar = originalVarIter.next();
-                    LogicalVariable newVar = newVarIter.next();
-                    for (SimilarAggregatesInfo sai : toReplaceSet) {
-                        for (AggregateExprInfo aei : sai.simAggs) {
-                            ILogicalExpression afce = aei.aggExprRef.getValue();
-                            afce.substituteVar(originalVar, newVar);
-                        }
-                    }
-                }
-                return new Pair<Boolean, ILogicalPlan>(true, null);
-            }
-        }
-    }
-
-    private ILogicalPlan fingIdenticalPlan(GroupByOperator newGbyOp, ILogicalPlan plan) throws AlgebricksException {
-        for (ILogicalPlan nestedPlan : newGbyOp.getNestedPlans()) {
-            if (IsomorphismUtilities.isOperatorIsomorphicPlan(plan, nestedPlan)) {
-                return nestedPlan;
-            }
-        }
-        return null;
-    }
-
-    private boolean tryToPushRoot(Mutable<ILogicalOperator> root, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
-            BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context,
-            List<Mutable<ILogicalOperator>> toPushAccumulate, Set<SimilarAggregatesInfo> toReplaceSet)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) root.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-        // Finds nested group-by if any.
-        AbstractLogicalOperator op3 = op2;
-        while (op3.getOperatorTag() != LogicalOperatorTag.GROUP && op3.getInputs().size() == 1) {
-            op3 = (AbstractLogicalOperator) op3.getInputs().get(0).getValue();
-        }
-
-        if (op3.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            AggregateOperator initAgg = (AggregateOperator) op1;
-            Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, toReplaceSet, context);
-            if (!pOpRef.first) {
-                return false;
-            }
-            Mutable<ILogicalOperator> opRef = pOpRef.second;
-            if (opRef != null) {
-                toPushAccumulate.add(opRef);
-            }
-            bi.modifyGbyMap.put(oldGbyOp, gbyVars);
-            return true;
-        } else {
-            GroupByOperator nestedGby = (GroupByOperator) op3;
-            List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
-            List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
-            concatGbyVars.addAll(gbyVars2);
-            for (ILogicalPlan p : nestedGby.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
-                    if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate,
-                            toReplaceSet)) {
-                        return false;
-                    }
-                }
-            }
-
-            /***
-             * Push the nested pipeline which provides the input to the nested group operator into newGbyOp (the combined gby op).
-             * The change is to fix asterixdb issue 782.
-             */
-            Mutable<ILogicalOperator> nestedGbyInputRef = nestedGby.getInputs().get(0);
-            Mutable<ILogicalOperator> startOfPipelineRef = nestedGbyInputRef;
-            if (startOfPipelineRef.getValue().getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-                return true;
-            }
-
-            // move down the nested pipeline to find the start of the pipeline right upon the nested-tuple-source
-            boolean hasIsNullFunction = OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) startOfPipelineRef
-                    .getValue());
-            while (startOfPipelineRef.getValue().getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
-                startOfPipelineRef = startOfPipelineRef.getValue().getInputs().get(0);
-                hasIsNullFunction = OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) startOfPipelineRef
-                        .getValue());
-            }
-            //keep the old nested-tuple-source
-            Mutable<ILogicalOperator> oldNts = startOfPipelineRef.getValue().getInputs().get(0);
-
-            //move down the nested op in the new gby operator
-            Mutable<ILogicalOperator> newGbyNestedOpRef = toPushAccumulate.get(0);
-            while (newGbyNestedOpRef.getValue().getInputs().get(0).getValue().getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
-                newGbyNestedOpRef = newGbyNestedOpRef.getValue().getInputs().get(0);
-            }
-
-            //insert the pipeline before nested gby into the new (combiner) gby's nested plan on top of the nested-tuple-source
-            startOfPipelineRef.getValue().getInputs().set(0, newGbyNestedOpRef.getValue().getInputs().get(0));
-            newGbyNestedOpRef.getValue().getInputs().set(0, nestedGbyInputRef);
-
-            //in the old gby operator, remove the nested pipeline since it is already pushed to the combiner gby
-            nestedGby.getInputs().set(0, oldNts);
-            List<LogicalVariable> aggProducedVars = new ArrayList<LogicalVariable>();
-            VariableUtilities.getProducedVariables(toPushAccumulate.get(0).getValue(), aggProducedVars);
-
-            if (hasIsNullFunction && aggProducedVars.size() != 0) {
-                // if the old nested pipeline contains a not-null-check, we need to convert it to a not-system-null-check in the non-local gby
-                processNullTest(context, nestedGby, aggProducedVars);
-            }
-
-            return true;
-        }
-    }
-
-    /**
-     * Deal with the case where the nested plan in the combiner gby operator has a null-test before invoking aggregation functions.
-     * 
-     * @param context
-     *            The optimization context.
-     * @param nestedGby
-     *            The nested gby operator in the global gby operator's subplan.
-     * @param firstAggVar
-     *            The first aggregation variable produced by the combiner gby.
-     */
-    protected abstract void processNullTest(IOptimizationContext context, GroupByOperator nestedGby,
-            List<LogicalVariable> aggregateVarsProducedByCombiner);
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
deleted file mode 100644
index 53ca903..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class BreakSelectIntoConjunctsRule implements IAlgebraicRewriteRule {
-
-    private List<Mutable<ILogicalExpression>> conjs = new ArrayList<Mutable<ILogicalExpression>>();
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
-            return false;
-        }
-        SelectOperator select = (SelectOperator) op;
-
-        ILogicalExpression cond = select.getCondition().getValue();
-
-        conjs.clear();
-        if (!cond.splitIntoConjuncts(conjs)) {
-            return false;
-        }
-
-        Mutable<ILogicalOperator> childOfSelect = select.getInputs().get(0);
-        boolean fst = true;
-        ILogicalOperator botOp = select;
-        ILogicalExpression firstExpr = null;
-        for (Mutable<ILogicalExpression> eRef : conjs) {
-            ILogicalExpression e = eRef.getValue();
-            if (fst) {
-                fst = false;
-                firstExpr = e;
-            } else {
-                SelectOperator newSelect = new SelectOperator(new MutableObject<ILogicalExpression>(e),
-                        select.getRetainNull(), select.getNullPlaceholderVariable());
-                List<Mutable<ILogicalOperator>> botInpList = botOp.getInputs();
-                botInpList.clear();
-                botInpList.add(new MutableObject<ILogicalOperator>(newSelect));
-                context.computeAndSetTypeEnvironmentForOperator(botOp);
-                botOp = newSelect;
-            }
-        }
-        botOp.getInputs().add(childOfSelect);
-        select.getCondition().setValue(firstExpr);
-        context.computeAndSetTypeEnvironmentForOperator(botOp);
-        context.computeAndSetTypeEnvironmentForOperator(select);
-
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
deleted file mode 100644
index 73fd966..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class ComplexJoinInferenceRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        ILogicalOperator op = opRef.getValue();
-        if (!(op instanceof AbstractScanOperator)) {
-            return false;
-        }
-
-        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op2;
-
-        Mutable<ILogicalOperator> opRef3 = subplan.getInputs().get(0);
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
-
-        if (op3.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
-                || op3.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            return false;
-        }
-
-        if (subplanHasFreeVariables(subplan)) {
-            return false;
-        }
-
-        HashSet<LogicalVariable> varsUsedInUnnest = new HashSet<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op, varsUsedInUnnest);
-
-        HashSet<LogicalVariable> producedInSubplan = new HashSet<LogicalVariable>();
-        VariableUtilities.getProducedVariables(subplan, producedInSubplan);
-
-        if (!producedInSubplan.containsAll(varsUsedInUnnest)) {
-            return false;
-        }
-
-        ntsToEtsInSubplan(subplan, context);
-        cleanupJoins(subplan);
-        InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
-        join.getInputs().add(opRef3);
-        opRef2.setValue(OperatorManipulationUtil.eliminateSingleSubplanOverEts(subplan));
-        join.getInputs().add(new MutableObject<ILogicalOperator>(op));
-        opRef.setValue(join);
-        context.computeAndSetTypeEnvironmentForOperator(join);
-        return true;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    private static void cleanupJoins(SubplanOperator s) {
-        for (ILogicalPlan p : s.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                cleanupJoins(r);
-            }
-        }
-    }
-
-    /** clean up joins that have one input branch that is empty tuple source */
-    private static void cleanupJoins(Mutable<ILogicalOperator> opRef) {
-        if (opRef.getValue() instanceof AbstractBinaryJoinOperator) {
-            for (Mutable<ILogicalOperator> inputRef : opRef.getValue().getInputs()) {
-                if (inputRef.getValue().getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-                    opRef.getValue().getInputs().remove(inputRef);
-                    opRef.setValue(opRef.getValue().getInputs().get(0).getValue());
-                    break;
-                }
-            }
-        }
-        for (Mutable<ILogicalOperator> inputRef : opRef.getValue().getInputs()) {
-            cleanupJoins(inputRef);
-        }
-    }
-
-    private static void ntsToEtsInSubplan(SubplanOperator s, IOptimizationContext context) throws AlgebricksException {
-        for (ILogicalPlan p : s.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                OperatorManipulationUtil.ntsToEts(r, context);
-            }
-        }
-    }
-
-    private static boolean subplanHasFreeVariables(SubplanOperator s) throws AlgebricksException {
-        for (ILogicalPlan p : s.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                if (OperatorPropertiesUtil.hasFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue())) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
deleted file mode 100644
index 910a8f9..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexUnnestToProductRule.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Complex rewrite rule for producing joins from unnests.
- * This rule is limited to creating left-deep trees.
- */
-public class ComplexUnnestToProductRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN
-                && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
-            return false;
-        }
-
-        //stop rewriting if the operators originates from a nested tuple source
-        if (insideSubplan(opRef)) {
-            return false;
-        }
-
-        // We may pull selects above the join we create in order to eliminate possible dependencies between
-        // the outer and inner input plans of the join.
-        List<ILogicalOperator> topSelects = new ArrayList<ILogicalOperator>();
-
-        // Keep track of the operators and used variables participating in the inner input plan.
-        HashSet<LogicalVariable> innerUsedVars = new HashSet<LogicalVariable>();
-        List<ILogicalOperator> innerOps = new ArrayList<ILogicalOperator>();
-        HashSet<LogicalVariable> outerUsedVars = new HashSet<LogicalVariable>();
-        List<ILogicalOperator> outerOps = new ArrayList<ILogicalOperator>();
-        innerOps.add(op);
-        VariableUtilities.getUsedVariables(op, innerUsedVars);
-
-        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
-
-        // Find an unnest or join and partition the plan between the first unnest and that operator into independent parts.
-        if (!findPlanPartition(op2, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, false)) {
-            // We could not find an unnest or join.
-            return false;
-        }
-        // The last operator must be an unnest or join.
-        AbstractLogicalOperator unnestOrJoin = (AbstractLogicalOperator) outerOps.get(outerOps.size() - 1);
-
-        ILogicalOperator outerRoot = null;
-        ILogicalOperator innerRoot = null;
-        EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
-        // If we found a join, simply use it as the outer root.
-        if (unnestOrJoin.getOperatorTag() != LogicalOperatorTag.INNERJOIN
-                && unnestOrJoin.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
-            // We've found a second unnest. First, sanity check that the unnest does not produce any vars that are used by the plan above (until the first unnest).
-            List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
-            VariableUtilities.getProducedVariables(unnestOrJoin, producedVars);
-            for (LogicalVariable producedVar : producedVars) {
-                if (innerUsedVars.contains(producedVar)) {
-                    return false;
-                }
-            }
-            // Continue finding a partitioning of the plan such that the inner and outer partitions are independent, in order to feed a join.
-            // Now, we look below the second unnest or join.
-            VariableUtilities.getUsedVariables(unnestOrJoin, outerUsedVars);
-            AbstractLogicalOperator unnestChild = (AbstractLogicalOperator) unnestOrJoin.getInputs().get(0).getValue();
-            if (!findPlanPartition(unnestChild, innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, true)) {
-                // We could not find a suitable partitioning.
-                return false;
-            }
-        }
-        innerRoot = buildOperatorChain(innerOps, ets, context);
-        context.computeAndSetTypeEnvironmentForOperator(innerRoot);
-        outerRoot = buildOperatorChain(outerOps, null, context);
-        context.computeAndSetTypeEnvironmentForOperator(outerRoot);
-
-        InnerJoinOperator product = new InnerJoinOperator(
-                new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
-        // Outer branch.
-        product.getInputs().add(new MutableObject<ILogicalOperator>(outerRoot));
-        // Inner branch.
-        product.getInputs().add(new MutableObject<ILogicalOperator>(innerRoot));
-        context.computeAndSetTypeEnvironmentForOperator(product);
-        // Put the selects on top of the join.
-        ILogicalOperator topOp = product;
-        if (!topSelects.isEmpty()) {
-            topOp = buildOperatorChain(topSelects, product, context);
-        }
-        // Plug the selects + product in the plan.
-        opRef.setValue(topOp);
-        context.computeAndSetTypeEnvironmentForOperator(topOp);
-        return true;
-    }
-
-    private ILogicalOperator buildOperatorChain(List<ILogicalOperator> ops, ILogicalOperator bottomOp,
-            IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator root = ops.get(0);
-        ILogicalOperator prevOp = root;
-        for (int i = 1; i < ops.size(); i++) {
-            ILogicalOperator inputOp = ops.get(i);
-            prevOp.getInputs().clear();
-            prevOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
-            prevOp = inputOp;
-        }
-        if (bottomOp != null) {
-            context.computeAndSetTypeEnvironmentForOperator(bottomOp);
-            prevOp.getInputs().clear();
-            prevOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomOp));
-        }
-        return root;
-    }
-
-    private boolean findPlanPartition(AbstractLogicalOperator op, HashSet<LogicalVariable> innerUsedVars,
-            HashSet<LogicalVariable> outerUsedVars, List<ILogicalOperator> innerOps, List<ILogicalOperator> outerOps,
-            List<ILogicalOperator> topSelects, boolean belowSecondUnnest) throws AlgebricksException {
-        if (belowSecondUnnest && innerUsedVars.isEmpty()) {
-            // Trivially joinable.
-            return true;
-        }
-        if (!belowSecondUnnest) {
-            // Bail on the following operators.
-            switch (op.getOperatorTag()) {
-                case AGGREGATE:
-                case SUBPLAN:
-                case GROUP:
-                case UNNEST_MAP:
-                    return false;
-            }
-        }
-        switch (op.getOperatorTag()) {
-            case UNNEST:
-            case DATASOURCESCAN: {
-                // We may have reached this state by descending through a subplan.
-                outerOps.add(op);
-                return true;
-            }
-            case INNERJOIN:
-            case LEFTOUTERJOIN: {
-                // Make sure that no variables that are live under this join are needed by the inner.
-                List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
-                VariableUtilities.getLiveVariables(op, liveVars);
-                for (LogicalVariable liveVar : liveVars) {
-                    if (innerUsedVars.contains(liveVar)) {
-                        return false;
-                    }
-                }
-                outerOps.add(op);
-                return true;
-            }
-            case SELECT: {
-                // Remember this select to pulling it above the join.
-                if (innerUsedVars.isEmpty()) {
-                    outerOps.add(op);
-                } else {
-                    topSelects.add(op);
-                }
-                break;
-            }
-            case PROJECT: {
-                // Throw away projects from the plan since we are pulling selects up.
-                break;
-            }
-            case EMPTYTUPLESOURCE:
-            case NESTEDTUPLESOURCE: {
-                if (belowSecondUnnest) {
-                    // We have successfully partitioned the plan into independent parts to be plugged into the join.
-                    return true;
-                } else {
-                    // We could not find a second unnest or a join.
-                    return false;
-                }
-            }
-            default: {
-                // The inner is trivially independent.
-                if (!belowSecondUnnest && innerUsedVars.isEmpty()) {
-                    outerOps.add(op);
-                    break;
-                }
-
-                // Examine produced vars to determine which partition uses them.
-                List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
-                VariableUtilities.getProducedVariables(op, producedVars);
-                int outerMatches = 0;
-                int innerMatches = 0;
-                for (LogicalVariable producedVar : producedVars) {
-                    if (outerUsedVars.contains(producedVar)) {
-                        outerMatches++;
-                    }
-                    if (innerUsedVars.contains(producedVar)) {
-                        innerMatches++;
-                    }
-                }
-
-                HashSet<LogicalVariable> targetUsedVars = null;
-                if (outerMatches == producedVars.size() && !producedVars.isEmpty()) {
-                    // All produced vars used by outer partition.
-                    outerOps.add(op);
-                    targetUsedVars = outerUsedVars;
-                }
-                if (innerMatches == producedVars.size() && !producedVars.isEmpty()) {
-                    // All produced vars used by inner partition.
-                    innerOps.add(op);
-                    targetUsedVars = innerUsedVars;
-                }
-                if (innerMatches == 0 && outerMatches == 0) {
-                    // Op produces variables that are not used in the part of the plan we've seen (or it doesn't produce any vars).
-                    // Try to figure out where it belongs by analyzing the used variables.
-                    List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
-                    VariableUtilities.getUsedVariables(op, usedVars);
-                    for (LogicalVariable usedVar : usedVars) {
-                        boolean canBreak = false;
-                        if (outerUsedVars.contains(usedVar)) {
-                            outerOps.add(op);
-                            targetUsedVars = outerUsedVars;
-                            canBreak = true;
-                        }
-                        if (innerUsedVars.contains(usedVar)) {
-                            innerOps.add(op);
-                            targetUsedVars = innerUsedVars;
-                            canBreak = true;
-                        }
-                        if (canBreak) {
-                            break;
-                        }
-                    }
-                    // TODO: For now we bail here, but we could remember such ops and determine their target partition at a later point.
-                    if (targetUsedVars == null) {
-                        return false;
-                    }
-                } else if (innerMatches != 0 && outerMatches != 0) {
-                    // The current operator produces variables that are used by both partitions, so the inner and outer are not independent and, therefore, we cannot create a join.
-                    // TODO: We may still be able to split the operator to create a viable partitioning.
-                    return false;
-                }
-                // Update used variables of partition that op belongs to.
-                if (op.hasNestedPlans() && op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-                    AbstractOperatorWithNestedPlans opWithNestedPlans = (AbstractOperatorWithNestedPlans) op;
-                    opWithNestedPlans.getUsedVariablesExceptNestedPlans(targetUsedVars);
-                } else {
-                    VariableUtilities.getUsedVariables(op, targetUsedVars);
-                }
-                break;
-            }
-        }
-        if (!op.hasInputs()) {
-            if (!belowSecondUnnest) {
-                // We could not find a second unnest or a join.
-                return false;
-            } else {
-                // We have successfully partitioned the plan into independent parts to be plugged into the join.
-                return true;
-            }
-        }
-        return findPlanPartition((AbstractLogicalOperator) op.getInputs().get(0).getValue(), innerUsedVars,
-                outerUsedVars, innerOps, outerOps, topSelects, belowSecondUnnest);
-    }
-
-    /**
-     * check whether the operator is inside a sub-plan
-     * 
-     * @param nestedRootRef
-     * @return true-if it is; false otherwise.
-     */
-    private boolean insideSubplan(Mutable<ILogicalOperator> nestedRootRef) {
-        AbstractLogicalOperator nestedRoot = (AbstractLogicalOperator) nestedRootRef.getValue();
-        if (nestedRoot.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            return true;
-        }
-        List<Mutable<ILogicalOperator>> inputs = nestedRoot.getInputs();
-        for (Mutable<ILogicalOperator> input : inputs) {
-            if (insideSubplan(input)) {
-                return true;
-            }
-        }
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
deleted file mode 100644
index 4da332c..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-            return false;
-        }
-        AssignOperator assign1 = (AssignOperator) op;
-
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) assign1.getInputs().get(0).getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-            return false;
-        }
-        AssignOperator assign2 = (AssignOperator) op2;
-
-        HashSet<LogicalVariable> used1 = new HashSet<LogicalVariable>();
-        VariableUtilities.getUsedVariables(assign1, used1);
-        for (LogicalVariable v2 : assign2.getVariables()) {
-            if (used1.contains(v2)) {
-                return false;
-            }
-        }
-
-        assign1.getVariables().addAll(assign2.getVariables());
-        assign1.getExpressions().addAll(assign2.getExpressions());
-
-        Mutable<ILogicalOperator> botOpRef = assign2.getInputs().get(0);
-        List<Mutable<ILogicalOperator>> asgnInpList = assign1.getInputs();
-        asgnInpList.clear();
-        asgnInpList.add(botOpRef);
-        context.computeAndSetTypeEnvironmentForOperator(assign1);
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
deleted file mode 100644
index e57bd3c..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Matches the following operator pattern:
- * (select) <-- ((assign)* <-- (select)*)+
- * 
- * Consolidates the selects to:
- * (select) <-- (assign)*
- *
- */
-public class ConsolidateSelectsRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-    	AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
-            return false;
-        }
-        SelectOperator firstSelect = (SelectOperator) op;
-
-        IFunctionInfo andFn = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND);
-        // New conjuncts for consolidated select.
-        AbstractFunctionCallExpression conj = null;        
-        AbstractLogicalOperator topMostOp = null;
-        AbstractLogicalOperator selectParent = null;
-        AbstractLogicalOperator nextSelect = firstSelect;
-		do {
-        	// Skip through assigns.
-            do {
-            	selectParent = nextSelect;
-            	nextSelect = (AbstractLogicalOperator) selectParent.getInputs().get(0).getValue();
-            } while (nextSelect.getOperatorTag() == LogicalOperatorTag.ASSIGN);
-            // Stop if the child op is not a select.
-            if (nextSelect.getOperatorTag() != LogicalOperatorTag.SELECT) {
-        		break;
-        	}
-            // Remember the top-most op that we are not removing.
-            topMostOp = selectParent;
-            
-            // Initialize the new conjuncts, if necessary.
-            if (conj == null) {
-            	conj = new ScalarFunctionCallExpression(andFn);
-            	// Add the first select's condition.
-            	conj.getArguments().add(new MutableObject<ILogicalExpression>(firstSelect.getCondition().getValue()));
-            }
-            
-            // Consolidate all following selects.
-            do {
-                // Add the condition nextSelect to the new list of conjuncts.
-                conj.getArguments().add(((SelectOperator) nextSelect).getCondition());
-                selectParent = nextSelect;
-                nextSelect = (AbstractLogicalOperator) nextSelect.getInputs().get(0).getValue();
-            } while (nextSelect.getOperatorTag() == LogicalOperatorTag.SELECT);
-            
-            // Hook up the input of the top-most remaining op if necessary.
-            if (topMostOp.getOperatorTag() == LogicalOperatorTag.ASSIGN || topMostOp == firstSelect) {
-            	topMostOp.getInputs().set(0, selectParent.getInputs().get(0));
-            }
-            
-            // Prepare for next iteration.
-            nextSelect = selectParent;
-        } while (true);
-		
-		// Did we consolidate any selects?
-        if (conj == null) {
-        	return false;
-        }
-        
-        // Set the new conjuncts.
-        firstSelect.getCondition().setValue(conj);
-        context.computeAndSetTypeEnvironmentForOperator(firstSelect);
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java
deleted file mode 100644
index 2f080fb..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class CopyLimitDownRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.LIMIT) {
-            return false;
-        }
-        LimitOperator limitOp = (LimitOperator) op;
-        if (!limitOp.isTopmostLimitOp()) {
-            return false;
-        }
-
-        List<LogicalVariable> limitUsedVars = new ArrayList<>();
-        VariableUtilities.getUsedVariables(limitOp, limitUsedVars);
-
-        Mutable<ILogicalOperator> safeOpRef = null;
-        Mutable<ILogicalOperator> candidateOpRef = limitOp.getInputs().get(0);
-
-        List<LogicalVariable> candidateProducedVars = new ArrayList<>();
-        while (true) {
-            candidateProducedVars.clear();
-            ILogicalOperator candidateOp = candidateOpRef.getValue();
-            LogicalOperatorTag candidateOpTag = candidateOp.getOperatorTag();
-            if (candidateOp.getInputs().size() > 1 || !candidateOp.isMap()
-                    || candidateOpTag == LogicalOperatorTag.SELECT || candidateOpTag == LogicalOperatorTag.LIMIT
-                    || !OperatorPropertiesUtil.disjoint(limitUsedVars, candidateProducedVars)) {
-                break;
-            }
-
-            safeOpRef = candidateOpRef;
-            candidateOpRef = safeOpRef.getValue().getInputs().get(0);
-        }
-
-        if (safeOpRef != null) {
-            ILogicalOperator safeOp = safeOpRef.getValue();
-            Mutable<ILogicalOperator> unsafeOpRef = safeOp.getInputs().get(0);
-            ILogicalOperator unsafeOp = unsafeOpRef.getValue();
-            LimitOperator limitCloneOp = null;
-            if (limitOp.getOffset().getValue() == null) {
-                limitCloneOp = new LimitOperator(limitOp.getMaxObjects().getValue(), false);
-            } else {
-                IFunctionInfo finfoAdd = context.getMetadataProvider().lookupFunction(
-                        AlgebricksBuiltinFunctions.NUMERIC_ADD);
-                List<Mutable<ILogicalExpression>> addArgs = new ArrayList<>();
-                addArgs.add(new MutableObject<ILogicalExpression>(limitOp.getMaxObjects().getValue().cloneExpression()));
-                addArgs.add(new MutableObject<ILogicalExpression>(limitOp.getOffset().getValue().cloneExpression()));
-                ScalarFunctionCallExpression maxPlusOffset = new ScalarFunctionCallExpression(finfoAdd, addArgs);
-                limitCloneOp = new LimitOperator(maxPlusOffset, false);
-            }
-            limitCloneOp.setPhysicalOperator(new StreamLimitPOperator());
-            limitCloneOp.getInputs().add(new MutableObject<ILogicalOperator>(unsafeOp));
-            limitCloneOp.setExecutionMode(unsafeOp.getExecutionMode());
-            limitCloneOp.recomputeSchema();
-            unsafeOpRef.setValue(limitCloneOp);
-            context.computeAndSetTypeEnvironmentForOperator(limitCloneOp);
-            context.addToDontApplySet(this, limitOp);
-        }
-
-        return safeOpRef != null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java
deleted file mode 100644
index e93fdd1..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This rule lift out the aggregate operator out from a group-by operator
- * if the gby operator groups-by on empty key, e.g., the group-by variables are empty.
- * 
- * @author yingyib
- */
-public class EliminateGroupByEmptyKeyRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
-            return false;
-        }
-        GroupByOperator groupOp = (GroupByOperator) op;
-        List<LogicalVariable> groupVars = groupOp.getGbyVarList();
-        if (groupVars.size() > 0) {
-            return false;
-        }
-        List<ILogicalPlan> nestedPlans = groupOp.getNestedPlans();
-        if (nestedPlans.size() > 1) {
-            return false;
-        }
-        ILogicalPlan nestedPlan = nestedPlans.get(0);
-        if (nestedPlan.getRoots().size() > 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> topOpRef = nestedPlan.getRoots().get(0);
-        ILogicalOperator topOp = nestedPlan.getRoots().get(0).getValue();
-        Mutable<ILogicalOperator> nestedTupleSourceRef = getNestedTupleSourceReference(topOpRef);
-        /**
-         * connect nested top op into the plan
-         */
-        opRef.setValue(topOp);
-        /**
-         * connect child op into the plan
-         */
-        nestedTupleSourceRef.setValue(groupOp.getInputs().get(0).getValue());
-        return true;
-    }
-
-    private Mutable<ILogicalOperator> getNestedTupleSourceReference(Mutable<ILogicalOperator> nestedTopOperatorRef) {
-        Mutable<ILogicalOperator> currentOpRef = nestedTopOperatorRef;
-        while (currentOpRef.getValue().getInputs() != null && currentOpRef.getValue().getInputs().size() > 0) {
-            currentOpRef = currentOpRef.getValue().getInputs().get(0);
-        }
-        return currentOpRef;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
deleted file mode 100644
index 8a381f7..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.LinkedList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class EliminateSubplanRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    /**
-     * Eliminate Subplan above ETS
-     * and Subplan that has only ops. with one input and no free vars. (could we
-     * modify it to consider free vars which are sources of Unnest or Assign, if
-     * there are no aggregates?)
-     */
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op;
-
-        Mutable<ILogicalOperator> outerRef = subplan.getInputs().get(0);
-        AbstractLogicalOperator outerRefOp = (AbstractLogicalOperator) outerRef.getValue();
-        if (outerRefOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-            elimSubplanOverEts(opRef, context);
-            return true;
-        }
-        if (subplan.getNestedPlans().size() == 1 && subplan.getNestedPlans().get(0).getRoots().size() == 1
-                && !OperatorPropertiesUtil.hasFreeVariables(subplan)) {
-            if (elimOneSubplanWithNoFreeVars(opRef)) {
-                return true;
-            }
-        }
-
-        return false;
-    }
-
-    private boolean elimOneSubplanWithNoFreeVars(Mutable<ILogicalOperator> opRef) {
-        SubplanOperator subplan = (SubplanOperator) opRef.getValue();
-        AbstractLogicalOperator rootOp = (AbstractLogicalOperator) subplan.getNestedPlans().get(0).getRoots().get(0)
-                .getValue();
-        if (rootOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
-                || rootOp.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            opRef.setValue(subplan.getInputs().get(0).getValue());
-            return true;
-        } else {
-            AbstractLogicalOperator botOp = rootOp;
-            if (botOp.getInputs().size() != 1) {
-                return false;
-            }
-            do {
-                Mutable<ILogicalOperator> botRef = botOp.getInputs().get(0);
-                botOp = (AbstractLogicalOperator) botRef.getValue();
-                if (botOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-                    botRef.setValue(subplan.getInputs().get(0).getValue());
-                    opRef.setValue(rootOp);
-                    return true;
-                }
-            } while (botOp.getInputs().size() == 1);
-            return false;
-        }
-    }
-
-    private void elimSubplanOverEts(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx)
-            throws AlgebricksException {
-        SubplanOperator subplan = (SubplanOperator) opRef.getValue();
-        for (ILogicalPlan p : subplan.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                OperatorManipulationUtil.ntsToEts(r, ctx);
-            }
-        }
-        LinkedList<Mutable<ILogicalOperator>> allRoots = subplan.allRootsInReverseOrder();
-        if (allRoots.size() == 1) {
-            opRef.setValue(allRoots.get(0).getValue());
-        } else {
-            ILogicalOperator topOp = null;
-            for (Mutable<ILogicalOperator> r : allRoots) {
-                if (topOp == null) {
-                    topOp = r.getValue();
-                } else {
-                    LeftOuterJoinOperator j = new LeftOuterJoinOperator(new MutableObject<ILogicalExpression>(
-                            ConstantExpression.TRUE));
-                    j.getInputs().add(new MutableObject<ILogicalOperator>(topOp));
-                    j.getInputs().add(r);
-                    ctx.setOutputTypeEnvironment(j, j.computeOutputTypeEnvironment(ctx));
-                    topOp = j;
-                }
-            }
-            opRef.setValue(topOp);
-        }
-    }
-}

[31/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
new file mode 100644
index 0000000..46e9859
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class ScriptOperator extends AbstractLogicalOperator {
+
+    private ArrayList<LogicalVariable> inputVariables;
+    private ArrayList<LogicalVariable> outputVariables;
+    private IScriptDescription scriptDesc;
+
+    public ScriptOperator(IScriptDescription scriptDesc, ArrayList<LogicalVariable> inputVariables,
+            ArrayList<LogicalVariable> outputVariables) {
+        this.inputVariables = inputVariables;
+        this.outputVariables = outputVariables;
+        this.scriptDesc = scriptDesc;
+    }
+
+    public ArrayList<LogicalVariable> getInputVariables() {
+        return inputVariables;
+    }
+
+    public ArrayList<LogicalVariable> getOutputVariables() {
+        return outputVariables;
+    }
+
+    public IScriptDescription getScriptDescription() {
+        return scriptDesc;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.SCRIPT;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitScriptOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                for (LogicalVariable v : outputVariables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        this.schema = outputVariables;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getMetadataProvider());
+        for (Pair<LogicalVariable, Object> p : scriptDesc.getVarTypePairs()) {
+            env.setVarType(p.first, p.second);
+        }
+        return env;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
new file mode 100644
index 0000000..5aa229c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SelectOperator extends AbstractLogicalOperator {
+    private final Mutable<ILogicalExpression> condition;
+    private final boolean retainNull;
+    private final LogicalVariable nullPlaceholderVar;
+
+    public SelectOperator(Mutable<ILogicalExpression> condition, boolean retainNull, LogicalVariable nullPlaceholderVar) {
+        this.condition = condition;
+        this.retainNull = retainNull;
+        this.nullPlaceholderVar = nullPlaceholderVar;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.SELECT;
+    }
+
+    public Mutable<ILogicalExpression> getCondition() {
+        return condition;
+    }
+
+    public boolean getRetainNull() {
+        return retainNull;
+    }
+
+    public LogicalVariable getNullPlaceholderVariable() throws AlgebricksException {
+        return nullPlaceholderVar;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return visitor.transform(condition);
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitSelectOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return true;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[1];
+        envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
+        PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+        if (condition.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+            AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) condition.getValue();
+            if (f1.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
+                ILogicalExpression a1 = f1.getArguments().get(0).getValue();
+                if (a1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                    AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
+                    if (f2.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
+                        ILogicalExpression a2 = f2.getArguments().get(0).getValue();
+                        if (a2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                            LogicalVariable var = ((VariableReferenceExpression) a2).getVariableReference();
+                            env.getNonNullVariables().add(var);
+                        }
+                    }
+                }
+            }
+        }
+        return env;
+    }
+
+    @Override
+    public boolean requiresVariableReferenceExpressions() {
+        return false;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
new file mode 100644
index 0000000..84ad33b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SinkOperator extends AbstractLogicalOperator {
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        schema = new ArrayList<LogicalVariable>();
+        for (int i = 0; i < inputs.size(); i++) {
+            for (LogicalVariable v : inputs.get(i).getValue().getSchema()) {
+                if (!schema.contains(v))
+                	schema.add(v);
+            }
+
+        }
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitSinkOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+      ITypeEnvPointer[] envPointers = new ITypeEnvPointer[inputs.size()];
+      for (int i = 0; i < inputs.size(); i++) {
+          envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
+      }
+      PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+              ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+      return env;
+
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.SINK;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
new file mode 100644
index 0000000..c0a01b3
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class SubplanOperator extends AbstractOperatorWithNestedPlans {
+
+    public SubplanOperator() {
+        super();
+    }
+
+    public SubplanOperator(List<ILogicalPlan> plans) {
+        super(plans);
+    }
+
+    public SubplanOperator(ILogicalOperator planRoot) {
+        ArrayList<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>(1);
+        roots.add(new MutableObject<ILogicalOperator>(planRoot));
+        nestedPlans.add(new ALogicalPlanImpl(roots));
+    }
+
+    public void setRootOp(Mutable<ILogicalOperator> opRef) {
+        ILogicalPlan p = new ALogicalPlanImpl(opRef);
+        nestedPlans.add(p);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+        // do nothing
+        return false;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.SUBPLAN;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ADDNEWVARIABLES;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitSubplanOperator(this, arg);
+    }
+
+    @Override
+    public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+        // do nothing
+    }
+
+    @Override
+    public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+        // do nothing
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        int n = 0;
+        for (ILogicalPlan p : nestedPlans) {
+            n += p.getRoots().size();
+        }
+        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n + 1];
+        envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
+        int i = 1;
+        for (ILogicalPlan p : nestedPlans) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
+                i++;
+            }
+        }
+        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+                ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java
new file mode 100644
index 0000000..82c1035
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class TokenizeOperator extends AbstractLogicalOperator {
+
+    private final IDataSourceIndex<?, ?> dataSourceIndex;
+    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
+    private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
+    // logical variables for token, and number of token
+    private final List<LogicalVariable> tokenizeVars;
+    private final Mutable<ILogicalExpression> filterExpr;
+    private final Kind operation;
+    private final boolean bulkload;
+    private final boolean isPartitioned;
+    // contains the type for each variable in the tokenizeVars
+    private final List<Object> tokenizeVarTypes;
+    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+
+    public TokenizeOperator(IDataSourceIndex<?, ?> dataSourceIndex,
+            List<Mutable<ILogicalExpression>> primaryKeyExprs,
+            List<Mutable<ILogicalExpression>> secondaryKeyExprs,
+            List<LogicalVariable> tokenizeVars,
+            Mutable<ILogicalExpression> filterExpr, Kind operation,
+            boolean bulkload, boolean isPartitioned,
+            List<Object> tokenizeVarTypes) {
+        this.dataSourceIndex = dataSourceIndex;
+        this.primaryKeyExprs = primaryKeyExprs;
+        this.secondaryKeyExprs = secondaryKeyExprs;
+        this.tokenizeVars = tokenizeVars;
+        this.filterExpr = filterExpr;
+        this.operation = operation;
+        this.bulkload = bulkload;
+        this.isPartitioned = isPartitioned;
+        this.tokenizeVarTypes = tokenizeVarTypes;
+    }
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+        schema.addAll(tokenizeVars);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(
+            ILogicalExpressionReferenceTransform visitor)
+            throws AlgebricksException {
+        boolean b = false;
+        for (int i = 0; i < primaryKeyExprs.size(); i++) {
+            if (visitor.transform(primaryKeyExprs.get(i))) {
+                b = true;
+            }
+        }
+        for (int i = 0; i < secondaryKeyExprs.size(); i++) {
+            if (visitor.transform(secondaryKeyExprs.get(i))) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg)
+            throws AlgebricksException {
+        return visitor.visitTokenizeOperator(this, arg);
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target,
+                    IOperatorSchema... sources) throws AlgebricksException {
+                target.addAllVariables(sources[0]);
+                for (LogicalVariable v : tokenizeVars) {
+                    target.addVariable(v);
+                }
+            }
+        };
+
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.TOKENIZE;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(
+            ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+
+        // If the secondary index is not length-partitioned, create one new
+        // output variable - token.
+        // If the secondary index is length-partitioned, create two new output
+        // variables - token, number of token.
+        // The type of this variable will be the same as the type of the
+        // secondary key. If the secondary is list type, then the element type
+        // of the list.
+        // Along with PK, the tokenizer will generate [token, number of tokens,
+        // PK] pairs.
+
+        for (int i = 0; i < tokenizeVars.size(); i++) {
+            env.setVarType(tokenizeVars.get(i), tokenizeVarTypes.get(i));
+        }
+
+        return env;
+    }
+
+    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
+        return primaryKeyExprs;
+    }
+
+    public IDataSourceIndex<?, ?> getDataSourceIndex() {
+        return dataSourceIndex;
+    }
+
+    public List<Mutable<ILogicalExpression>> getSecondaryKeyExpressions() {
+        return secondaryKeyExprs;
+    }
+
+    public List<LogicalVariable> getTokenizeVars() {
+        return tokenizeVars;
+    }
+
+    public Mutable<ILogicalExpression> getFilterExpression() {
+        return filterExpr;
+    }
+
+    public Kind getOperation() {
+        return operation;
+    }
+
+    public boolean isBulkload() {
+        return bulkload;
+    }
+
+    public boolean isPartitioned() {
+        return isPartitioned;
+    }
+
+    public List<Object> getTokenizeVarTypes() {
+        return tokenizeVarTypes;
+    }
+
+    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+        this.additionalFilteringExpressions = additionalFilteringExpressions;
+    }
+
+    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+        return additionalFilteringExpressions;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
new file mode 100644
index 0000000..97a77ec
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class UnionAllOperator extends AbstractLogicalOperator {
+
+    // (left-var, right-var, out-var)
+    private List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap;
+
+    public UnionAllOperator(List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap) {
+        this.varMap = varMap;
+    }
+
+    public List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> getVariableMappings() {
+        return varMap;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.UNIONALL;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+                    target.addVariable(t.third);
+                }
+            }
+        };
+
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitUnionOperator(this, arg);
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        for (LogicalVariable v1 : inputs.get(0).getValue().getSchema()) {
+            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+                if (t.first.equals(v1)) {
+                    schema.add(t.third);
+                } else {
+                    schema.add(v1);
+                }
+            }
+        }
+        for (LogicalVariable v2 : inputs.get(1).getValue().getSchema()) {
+            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+                if (t.second.equals(v2)) {
+                    schema.add(t.third);
+                } else {
+                    schema.add(v2);
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getMetadataProvider());
+        IVariableTypeEnvironment envLeft = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
+        if (envLeft == null) {
+            throw new AlgebricksException("Left input types for union operator are not computed.");
+        }
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+            Object t1 = envLeft.getVarType(t.first);
+            if (t1 == null) {
+                throw new AlgebricksException("Failed typing union operator: no type for variable " + t.first);
+            }
+            env.setVarType(t.third, t1);
+        }
+        return env;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
new file mode 100644
index 0000000..b7a56be
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+/**
+ * This operator may go away after we add indexes to Algebricks.
+ */
+public class UnnestMapOperator extends AbstractUnnestOperator {
+
+    private final List<Object> variableTypes; // TODO: get rid of this and  deprecate UnnestMap
+    private boolean propagateInput;
+
+    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+    private List<LogicalVariable> minFilterVars;
+    private List<LogicalVariable> maxFilterVars;
+
+    public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
+            List<Object> variableTypes, boolean propagateInput) {
+        super(variables, expression);
+        this.variableTypes = variableTypes;
+        this.propagateInput = propagateInput;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.UNNEST_MAP;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitUnnestMapOperator(this, arg);
+    }
+
+    /**
+     * UnnestMap doesn't propagate input variables, because currently it is only
+     * used to search indexes. In the future, it would be nice to have the
+     * choice to propagate input variables or not.
+     */
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (propagateInput) {
+                    target.addAllVariables(sources[0]);
+                }
+                for (LogicalVariable v : variables) {
+                    target.addVariable(v);
+                }
+            }
+        };
+    }
+
+    public List<Object> getVariableTypes() {
+        return variableTypes;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = null;
+        if (propagateInput) {
+            env = createPropagatingAllInputsTypeEnvironment(ctx);
+        } else {
+            env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMetadataProvider());
+        }
+        int n = variables.size();
+        for (int i = 0; i < n; i++) {
+            env.setVarType(variables.get(i), variableTypes.get(i));
+        }
+        return env;
+    }
+
+    public boolean propagatesInput() {
+        return propagateInput;
+    }
+
+    public List<LogicalVariable> getMinFilterVars() {
+        return minFilterVars;
+    }
+
+    public void setMinFilterVars(List<LogicalVariable> minFilterVars) {
+        this.minFilterVars = minFilterVars;
+    }
+
+    public List<LogicalVariable> getMaxFilterVars() {
+        return maxFilterVars;
+    }
+
+    public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) {
+        this.maxFilterVars = maxFilterVars;
+    }
+
+    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+        this.additionalFilteringExpressions = additionalFilteringExpressions;
+    }
+
+    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+        return additionalFilteringExpressions;
+    }
+
+    /*
+    @Override
+    public boolean isMap() {
+        return !propagateInput;
+    }
+    */
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
new file mode 100644
index 0000000..dba9e19
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingPositionWriter;
+
+public class UnnestOperator extends AbstractUnnestOperator {
+
+    private LogicalVariable positionalVariable;
+
+    /**
+     * Used to set the position offset for positional variable
+     */
+    private ILogicalExpression positionOffsetExpr;
+
+    /**
+     * Specify the writer of the positional variable
+     */
+    private IUnnestingPositionWriter positionWriter;
+
+    /**
+     * Specify the type of the positional variable
+     */
+    private Object positionalVariableType;
+
+    public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression) {
+        super(makeSingletonList(variable), expression);
+    }
+
+    public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression,
+            LogicalVariable positionalVariable, Object positionalVariableType, IUnnestingPositionWriter positionWriter) {
+        this(variable, expression);
+        this.setPositionalVariable(positionalVariable);
+        this.setPositionalVariableType(positionalVariableType);
+        this.setPositionWriter(positionWriter);
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.UNNEST;
+    }
+
+    public LogicalVariable getVariable() {
+        return variables.get(0);
+    }
+
+    public void setPositionalVariable(LogicalVariable positionalVariable) {
+        this.positionalVariable = positionalVariable;
+    }
+
+    public LogicalVariable getPositionalVariable() {
+        return positionalVariable;
+    }
+
+    public void setPositionWriter(IUnnestingPositionWriter positionWriter) {
+        this.positionWriter = positionWriter;
+    }
+
+    public IUnnestingPositionWriter getPositionWriter() {
+        return positionalVariable != null ? positionWriter : null;
+    }
+
+    public void setPositionalVariableType(Object positionalVariableType) {
+        this.positionalVariableType = positionalVariableType;
+    }
+
+    public Object getPositionalVariableType() {
+        return positionalVariableType;
+    }
+
+    public void setPositionOffsetExpr(ILogicalExpression posOffsetExpr) {
+        this.positionOffsetExpr = posOffsetExpr;
+    }
+
+    public ILogicalExpression getPositionOffsetExpr() {
+        return this.positionOffsetExpr;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitUnnestOperator(this, arg);
+    }
+
+    private static <E> ArrayList<E> makeSingletonList(E item) {
+        ArrayList<E> array = new ArrayList<E>(1);
+        array.add(item);
+        return array;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+        Object t = env.getType(expression.getValue());
+        env.setVarType(variables.get(0), t);
+        if (positionalVariable != null) {
+            env.setVarType(positionalVariable, positionalVariableType);
+        }
+        return env;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return new VariablePropagationPolicy() {
+
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (sources.length > 0) {
+                    target.addAllVariables(sources[0]);
+                }
+                for (LogicalVariable v : variables) {
+                    target.addVariable(v);
+                }
+                if (positionalVariable != null) {
+                    target.addVariable(positionalVariable);
+                }
+            }
+        };
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
new file mode 100644
index 0000000..841f9c6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class UpdateOperator extends AbstractLogicalOperator {
+
+    @Override
+    public void recomputeSchema() throws AlgebricksException {
+        // TODO Auto-generated method stub
+
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public boolean isMap() {
+        // TODO Auto-generated method stub
+        return false;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
new file mode 100644
index 0000000..e1c1640
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class WriteOperator extends AbstractLogicalOperator {
+    private List<Mutable<ILogicalExpression>> expressions;
+    private IDataSink dataSink;
+
+    public WriteOperator(List<Mutable<ILogicalExpression>> expressions, IDataSink dataSink) {
+        this.expressions = expressions;
+        this.dataSink = dataSink;
+    }
+
+    public List<Mutable<ILogicalExpression>> getExpressions() {
+        return expressions;
+    }
+
+    public IDataSink getDataSink() {
+        return dataSink;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.WRITE;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitWriteOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean modif = false;
+        for (int i = 0; i < expressions.size(); i++) {
+            boolean b = visitor.transform(expressions.get(i));
+            if (b) {
+                modif = true;
+            }
+        }
+        return modif;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return false; // actually depends on the physical op.
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
new file mode 100644
index 0000000..ba402bc
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class WriteResultOperator extends AbstractLogicalOperator {
+
+    private IDataSource<?> dataSource;
+    private Mutable<ILogicalExpression> payloadExpr;
+    private List<Mutable<ILogicalExpression>> keyExprs;
+    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
+
+    public WriteResultOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
+            List<Mutable<ILogicalExpression>> keyExprs) {
+        this.dataSource = dataSource;
+        this.payloadExpr = payload;
+        this.keyExprs = keyExprs;
+    }
+
+    public IDataSource<?> getDataSource() {
+        return dataSource;
+    }
+
+    public Mutable<ILogicalExpression> getPayloadExpression() {
+        return payloadExpr;
+    }
+
+    public List<Mutable<ILogicalExpression>> getKeyExpressions() {
+        return keyExprs;
+    }
+
+    @Override
+    public LogicalOperatorTag getOperatorTag() {
+        return LogicalOperatorTag.WRITE_RESULT;
+    }
+
+    @Override
+    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+        return visitor.visitWriteResultOperator(this, arg);
+    }
+
+    @Override
+    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+        boolean b = visitor.transform(payloadExpr);
+        for (int i = 0; i < keyExprs.size(); i++) {
+            if (visitor.transform(keyExprs.get(i))) {
+                b = true;
+            }
+        }
+        return b;
+    }
+
+    @Override
+    public VariablePropagationPolicy getVariablePropagationPolicy() {
+        return VariablePropagationPolicy.ALL;
+    }
+
+    @Override
+    public boolean isMap() {
+        return false;
+    }
+
+    @Override
+    public void recomputeSchema() {
+        schema = new ArrayList<LogicalVariable>();
+        schema.addAll(inputs.get(0).getValue().getSchema());
+    }
+
+    @Override
+    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+        return createPropagatingAllInputsTypeEnvironment(ctx);
+    }
+
+    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
+        this.additionalFilteringExpressions = additionalFilteringExpressions;
+    }
+
+    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
+        return additionalFilteringExpressions;
+    }
+
+}

[45/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
deleted file mode 100644
index b7a56be..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-/**
- * This operator may go away after we add indexes to Algebricks.
- */
-public class UnnestMapOperator extends AbstractUnnestOperator {
-
-    private final List<Object> variableTypes; // TODO: get rid of this and  deprecate UnnestMap
-    private boolean propagateInput;
-
-    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
-    private List<LogicalVariable> minFilterVars;
-    private List<LogicalVariable> maxFilterVars;
-
-    public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
-            List<Object> variableTypes, boolean propagateInput) {
-        super(variables, expression);
-        this.variableTypes = variableTypes;
-        this.propagateInput = propagateInput;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.UNNEST_MAP;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitUnnestMapOperator(this, arg);
-    }
-
-    /**
-     * UnnestMap doesn't propagate input variables, because currently it is only
-     * used to search indexes. In the future, it would be nice to have the
-     * choice to propagate input variables or not.
-     */
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                if (propagateInput) {
-                    target.addAllVariables(sources[0]);
-                }
-                for (LogicalVariable v : variables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-    }
-
-    public List<Object> getVariableTypes() {
-        return variableTypes;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = null;
-        if (propagateInput) {
-            env = createPropagatingAllInputsTypeEnvironment(ctx);
-        } else {
-            env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getMetadataProvider());
-        }
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            env.setVarType(variables.get(i), variableTypes.get(i));
-        }
-        return env;
-    }
-
-    public boolean propagatesInput() {
-        return propagateInput;
-    }
-
-    public List<LogicalVariable> getMinFilterVars() {
-        return minFilterVars;
-    }
-
-    public void setMinFilterVars(List<LogicalVariable> minFilterVars) {
-        this.minFilterVars = minFilterVars;
-    }
-
-    public List<LogicalVariable> getMaxFilterVars() {
-        return maxFilterVars;
-    }
-
-    public void setMaxFilterVars(List<LogicalVariable> maxFilterVars) {
-        this.maxFilterVars = maxFilterVars;
-    }
-
-    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
-        this.additionalFilteringExpressions = additionalFilteringExpressions;
-    }
-
-    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
-        return additionalFilteringExpressions;
-    }
-
-    /*
-    @Override
-    public boolean isMap() {
-        return !propagateInput;
-    }
-    */
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
deleted file mode 100644
index dba9e19..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingPositionWriter;
-
-public class UnnestOperator extends AbstractUnnestOperator {
-
-    private LogicalVariable positionalVariable;
-
-    /**
-     * Used to set the position offset for positional variable
-     */
-    private ILogicalExpression positionOffsetExpr;
-
-    /**
-     * Specify the writer of the positional variable
-     */
-    private IUnnestingPositionWriter positionWriter;
-
-    /**
-     * Specify the type of the positional variable
-     */
-    private Object positionalVariableType;
-
-    public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression) {
-        super(makeSingletonList(variable), expression);
-    }
-
-    public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression,
-            LogicalVariable positionalVariable, Object positionalVariableType, IUnnestingPositionWriter positionWriter) {
-        this(variable, expression);
-        this.setPositionalVariable(positionalVariable);
-        this.setPositionalVariableType(positionalVariableType);
-        this.setPositionWriter(positionWriter);
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.UNNEST;
-    }
-
-    public LogicalVariable getVariable() {
-        return variables.get(0);
-    }
-
-    public void setPositionalVariable(LogicalVariable positionalVariable) {
-        this.positionalVariable = positionalVariable;
-    }
-
-    public LogicalVariable getPositionalVariable() {
-        return positionalVariable;
-    }
-
-    public void setPositionWriter(IUnnestingPositionWriter positionWriter) {
-        this.positionWriter = positionWriter;
-    }
-
-    public IUnnestingPositionWriter getPositionWriter() {
-        return positionalVariable != null ? positionWriter : null;
-    }
-
-    public void setPositionalVariableType(Object positionalVariableType) {
-        this.positionalVariableType = positionalVariableType;
-    }
-
-    public Object getPositionalVariableType() {
-        return positionalVariableType;
-    }
-
-    public void setPositionOffsetExpr(ILogicalExpression posOffsetExpr) {
-        this.positionOffsetExpr = posOffsetExpr;
-    }
-
-    public ILogicalExpression getPositionOffsetExpr() {
-        return this.positionOffsetExpr;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitUnnestOperator(this, arg);
-    }
-
-    private static <E> ArrayList<E> makeSingletonList(E item) {
-        ArrayList<E> array = new ArrayList<E>(1);
-        array.add(item);
-        return array;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
-        Object t = env.getType(expression.getValue());
-        env.setVarType(variables.get(0), t);
-        if (positionalVariable != null) {
-            env.setVarType(positionalVariable, positionalVariableType);
-        }
-        return env;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                if (sources.length > 0) {
-                    target.addAllVariables(sources[0]);
-                }
-                for (LogicalVariable v : variables) {
-                    target.addVariable(v);
-                }
-                if (positionalVariable != null) {
-                    target.addVariable(positionalVariable);
-                }
-            }
-        };
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
deleted file mode 100644
index 841f9c6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class UpdateOperator extends AbstractLogicalOperator {
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        // TODO Auto-generated method stub
-
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public boolean isMap() {
-        // TODO Auto-generated method stub
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
deleted file mode 100644
index e1c1640..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class WriteOperator extends AbstractLogicalOperator {
-    private List<Mutable<ILogicalExpression>> expressions;
-    private IDataSink dataSink;
-
-    public WriteOperator(List<Mutable<ILogicalExpression>> expressions, IDataSink dataSink) {
-        this.expressions = expressions;
-        this.dataSink = dataSink;
-    }
-
-    public List<Mutable<ILogicalExpression>> getExpressions() {
-        return expressions;
-    }
-
-    public IDataSink getDataSink() {
-        return dataSink;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.WRITE;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitWriteOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean modif = false;
-        for (int i = 0; i < expressions.size(); i++) {
-            boolean b = visitor.transform(expressions.get(i));
-            if (b) {
-                modif = true;
-            }
-        }
-        return modif;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return false; // actually depends on the physical op.
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
deleted file mode 100644
index ba402bc..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class WriteResultOperator extends AbstractLogicalOperator {
-
-    private IDataSource<?> dataSource;
-    private Mutable<ILogicalExpression> payloadExpr;
-    private List<Mutable<ILogicalExpression>> keyExprs;
-    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
-
-    public WriteResultOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
-            List<Mutable<ILogicalExpression>> keyExprs) {
-        this.dataSource = dataSource;
-        this.payloadExpr = payload;
-        this.keyExprs = keyExprs;
-    }
-
-    public IDataSource<?> getDataSource() {
-        return dataSource;
-    }
-
-    public Mutable<ILogicalExpression> getPayloadExpression() {
-        return payloadExpr;
-    }
-
-    public List<Mutable<ILogicalExpression>> getKeyExpressions() {
-        return keyExprs;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.WRITE_RESULT;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitWriteResultOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean b = visitor.transform(payloadExpr);
-        for (int i = 0; i < keyExprs.size(); i++) {
-            if (visitor.transform(keyExprs.get(i))) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
-        this.additionalFilteringExpressions = additionalFilteringExpressions;
-    }
-
-    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
-        return additionalFilteringExpressions;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
deleted file mode 100644
index 1d7f134..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
+++ /dev/null
@@ -1,763 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-
-public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
-        ctx.putFDList(op, new ArrayList<FunctionalDependency>());
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
-        ctx.putEquivalenceClassMap(op, eqClasses);
-
-        // Propagates equivalence classes that from expressions.
-        // Note that an equivalence class can also contain expression members.
-        propagateEquivalenceFromExpressionsToVariables(eqClasses, op.getExpressions(), op.getVariables());
-
-        // Generates FDs.
-        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op, used);
-        List<FunctionalDependency> fds1 = getOrComputeFDs(inp1, ctx);
-        List<FunctionalDependency> eFds = new ArrayList<FunctionalDependency>(fds1.size());
-        for (FunctionalDependency fd : fds1) {
-            if (fd.getTail().containsAll(used)) {
-                List<LogicalVariable> hd = new ArrayList<LogicalVariable>(fd.getHead());
-                List<LogicalVariable> tl = new ArrayList<LogicalVariable>(fd.getTail());
-                tl.addAll(op.getVariables());
-                FunctionalDependency fd2 = new FunctionalDependency(hd, tl);
-                eFds.add(fd2);
-            } else {
-                eFds.add(fd);
-            }
-        }
-        ctx.putFDList(op, eFds);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
-        Map<LogicalVariable, EquivalenceClass> propagatedEqClasses = getOrComputeEqClasses(inp1, ctx);
-        eqClasses.putAll(propagatedEqClasses);
-        ctx.putEquivalenceClassMap(op, eqClasses);
-        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
-        ctx.putFDList(op, fds);
-        op.getDataSource().computeFDs(op.getVariables(), fds);
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator op0 = op.getInputs().get(0).getValue();
-        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
-        ctx.putFDList(op, functionalDependencies);
-        for (FunctionalDependency inherited : getOrComputeFDs(op0, ctx)) {
-            boolean isCoveredByDistinctByVars = true;
-            for (LogicalVariable v : inherited.getHead()) {
-                if (!op.isDistinctByVar(v)) {
-                    isCoveredByDistinctByVars = false;
-                }
-            }
-            if (isCoveredByDistinctByVars) {
-                List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
-                for (LogicalVariable v2 : inherited.getTail()) {
-                    if (op.isDistinctByVar(v2)) {
-                        newTail.add(v2);
-                    }
-                }
-                if (!newTail.isEmpty()) {
-                    List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead());
-                    FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
-                    functionalDependencies.add(newFd);
-                }
-            }
-        }
-        Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
-        List<Mutable<ILogicalExpression>> expressions = op.getExpressions();
-        for (Mutable<ILogicalExpression> pRef : expressions) {
-            ILogicalExpression p = pRef.getValue();
-            if (p.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) p;
-                gbySet.add(v.getVariableReference());
-            }
-        }
-        LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
-
-        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = getOrComputeEqClasses(op0, ctx);
-        ctx.putEquivalenceClassMap(op, equivalenceClasses);
-
-        lgp.normalizeGroupingColumns(equivalenceClasses, functionalDependencies);
-        Set<LogicalVariable> normSet = lgp.getColumnSet();
-        List<Mutable<ILogicalExpression>> newDistinctByList = new ArrayList<Mutable<ILogicalExpression>>();
-        for (Mutable<ILogicalExpression> p2Ref : expressions) {
-            ILogicalExpression p2 = p2Ref.getValue();
-            if (p2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression var2 = (VariableReferenceExpression) p2;
-                LogicalVariable v2 = var2.getVariableReference();
-                if (normSet.contains(v2)) {
-                    newDistinctByList.add(p2Ref);
-                }
-            } else {
-                newDistinctByList.add(p2Ref);
-            }
-        }
-        expressions.clear();
-        expressions.addAll(newDistinctByList);
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
-        ctx.putFDList(op, new ArrayList<FunctionalDependency>());
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
-        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
-        ctx.putEquivalenceClassMap(op, equivalenceClasses);
-        ctx.putFDList(op, functionalDependencies);
-
-        List<FunctionalDependency> inheritedFDs = new ArrayList<FunctionalDependency>();
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                ILogicalOperator op2 = r.getValue();
-                equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
-                inheritedFDs.addAll(getOrComputeFDs(op2, ctx));
-            }
-        }
-
-        ILogicalOperator op0 = op.getInputs().get(0).getValue();
-        inheritedFDs.addAll(getOrComputeFDs(op0, ctx));
-        Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx);
-        for (FunctionalDependency inherited : inheritedFDs) {
-            boolean isCoveredByGbyOrDecorVars = true;
-            List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead().size());
-            for (LogicalVariable v : inherited.getHead()) {
-                LogicalVariable vnew = getNewGbyVar(op, v);
-                if (vnew == null) {
-                    vnew = getNewDecorVar(op, v);
-                    if (vnew == null) {
-                        isCoveredByGbyOrDecorVars = false;
-                    }
-                    break;
-                }
-                newHead.add(vnew);
-            }
-
-            if (isCoveredByGbyOrDecorVars) {
-                List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
-                for (LogicalVariable v2 : inherited.getTail()) {
-                    LogicalVariable v3 = getNewGbyVar(op, v2);
-                    if (v3 != null) {
-                        newTail.add(v3);
-                    }
-                }
-                if (!newTail.isEmpty()) {
-                    FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
-                    functionalDependencies.add(newFd);
-                }
-            }
-        }
-
-        List<LogicalVariable> premiseGby = new LinkedList<LogicalVariable>();
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-            premiseGby.add(p.first);
-        }
-
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList();
-
-        LinkedList<LogicalVariable> conclDecor = new LinkedList<LogicalVariable>();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
-            conclDecor.add(GroupByOperator.getDecorVariable(p));
-        }
-        if (!conclDecor.isEmpty()) {
-            functionalDependencies.add(new FunctionalDependency(premiseGby, conclDecor));
-        }
-
-        Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                gbySet.add(v.getVariableReference());
-            }
-        }
-        LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
-        lgp.normalizeGroupingColumns(inheritedEcs, inheritedFDs);
-        Set<LogicalVariable> normSet = lgp.getColumnSet();
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newGbyList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
-        boolean changed = false;
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
-                LogicalVariable v2 = varRef.getVariableReference();
-                EquivalenceClass ec2 = inheritedEcs.get(v2);
-                LogicalVariable v3;
-                if (ec2 != null && !ec2.representativeIsConst()) {
-                    v3 = ec2.getVariableRepresentative();
-                } else {
-                    v3 = v2;
-                }
-                if (normSet.contains(v3)) {
-                    newGbyList.add(p);
-                } else {
-                    changed = true;
-                    decorList.add(p);
-                }
-            } else {
-                newGbyList.add(p);
-            }
-        }
-        if (changed) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Group-by list changed from "
-                    + GroupByOperator.veListToString(gByList) + " to " + GroupByOperator.veListToString(newGbyList)
-                    + ".\n");
-        }
-        gByList.clear();
-        gByList.addAll(newGbyList);
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
-        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
-        ctx.putEquivalenceClassMap(op, equivalenceClasses);
-        ctx.putFDList(op, functionalDependencies);
-        ILogicalOperator op0 = op.getInputs().get(0).getValue();
-        ILogicalOperator op1 = op.getInputs().get(1).getValue();
-        functionalDependencies.addAll(getOrComputeFDs(op0, ctx));
-        functionalDependencies.addAll(getOrComputeFDs(op1, ctx));
-        equivalenceClasses.putAll(getOrComputeEqClasses(op0, ctx));
-        equivalenceClasses.putAll(getOrComputeEqClasses(op1, ctx));
-        ILogicalExpression expr = op.getCondition().getValue();
-        expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
-        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
-        ctx.putEquivalenceClassMap(op, equivalenceClasses);
-        ctx.putFDList(op, functionalDependencies);
-        ILogicalOperator opLeft = op.getInputs().get(0).getValue();
-        ILogicalOperator opRight = op.getInputs().get(1).getValue();
-        functionalDependencies.addAll(getOrComputeFDs(opLeft, ctx));
-        functionalDependencies.addAll(getOrComputeFDs(opRight, ctx));
-        equivalenceClasses.putAll(getOrComputeEqClasses(opLeft, ctx));
-        equivalenceClasses.putAll(getOrComputeEqClasses(opRight, ctx));
-
-        Collection<LogicalVariable> leftSideVars;
-        if (opLeft.getSchema() == null) {
-            leftSideVars = new LinkedList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(opLeft, leftSideVars);
-            // actually, not all produced vars. are visible (due to projection)
-            // so using cached schema is better and faster
-        } else {
-            leftSideVars = opLeft.getSchema();
-        }
-        ILogicalExpression expr = op.getCondition().getValue();
-        expr.getConstraintsForOuterJoin(functionalDependencies, leftSideVars);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getDataSourceReference().getValue();
-        ILogicalOperator inp1 = op1.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
-        ctx.putEquivalenceClassMap(op, eqClasses);
-        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
-        if (op1.getOperatorTag() == LogicalOperatorTag.GROUP) {
-            GroupByOperator gby = (GroupByOperator) op1;
-            LinkedList<LogicalVariable> tail = new LinkedList<LogicalVariable>();
-            for (LogicalVariable v : gby.getGbyVarList()) {
-                tail.add(v);
-                // all values for gby vars. are the same
-            }
-            FunctionalDependency gbyfd = new FunctionalDependency(new LinkedList<LogicalVariable>(), tail);
-            fds.add(gbyfd);
-        }
-        ctx.putFDList(op, fds);
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        throw new NotImplementedException();
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClassesForUsedVars(op, ctx, op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
-        ctx.putFDList(op, new ArrayList<FunctionalDependency>());
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClassesForUsedVars(op, ctx, op.getInputVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator childOp = op.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = getOrComputeEqClasses(childOp, ctx);
-        ctx.putEquivalenceClassMap(op, equivalenceClasses);
-
-        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
-        ctx.putFDList(op, functionalDependencies);
-        functionalDependencies.addAll(getOrComputeFDs(childOp, ctx));
-        equivalenceClasses.putAll(getOrComputeEqClasses(childOp, ctx));
-        ILogicalExpression expr = op.getCondition().getValue();
-        expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
-        List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
-        ctx.putEquivalenceClassMap(op, equivalenceClasses);
-        ctx.putFDList(op, functionalDependencies);
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                ILogicalOperator op2 = r.getValue();
-                equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
-                functionalDependencies.addAll(getOrComputeFDs(op2, ctx));
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        setEmptyFDsEqClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        fdsEqClassesForAbstractUnnestOperator(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        fdsEqClassesForAbstractUnnestOperator(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        // propagateFDsAndEquivClasses(op, ctx);
-        setEmptyFDsEqClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        // propagateFDsAndEquivClasses(op, ctx);
-        setEmptyFDsEqClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        // propagateFDsAndEquivClasses(op, ctx);
-        setEmptyFDsEqClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        setEmptyFDsEqClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        setEmptyFDsEqClasses(op, ctx);
-        return null;
-    }
-
-    private void propagateFDsAndEquivClasses(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
-        ctx.putEquivalenceClassMap(op, eqClasses);
-        List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
-        ctx.putFDList(op, fds);
-    }
-
-    private Map<LogicalVariable, EquivalenceClass> getOrComputeEqClasses(ILogicalOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        Map<LogicalVariable, EquivalenceClass> eqClasses = ctx.getEquivalenceClassMap(op);
-        if (eqClasses == null) {
-            op.accept(this, ctx);
-            eqClasses = ctx.getEquivalenceClassMap(op);
-        }
-        return eqClasses;
-    }
-
-    private Map<LogicalVariable, EquivalenceClass> getOrCreateEqClasses(ILogicalOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        Map<LogicalVariable, EquivalenceClass> eqClasses = ctx.getEquivalenceClassMap(op);
-        if (eqClasses == null) {
-            eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
-            ctx.putEquivalenceClassMap(op, eqClasses);
-        }
-        return eqClasses;
-    }
-
-    private List<FunctionalDependency> getOrComputeFDs(ILogicalOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        List<FunctionalDependency> fds = ctx.getFDList(op);
-        if (fds == null) {
-            op.accept(this, ctx);
-            fds = ctx.getFDList(op);
-        }
-        return fds;
-    }
-
-    /***
-     * Propagated equivalent classes from the child to the current operator, based
-     * on the used variables of the current operator.
-     * 
-     * @param op
-     *            , the current operator
-     * @param ctx
-     *            , the optimization context which keeps track of all equivalent classes.
-     * @param usedVariables
-     *            , used variables.
-     * @throws AlgebricksException
-     */
-    private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx,
-            List<LogicalVariable> usedVariables) throws AlgebricksException {
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
-        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
-        ctx.putFDList(op, fds);
-
-        Map<LogicalVariable, EquivalenceClass> chldClasses = getOrComputeEqClasses(op2, ctx);
-
-        // Propagate equivalent classes that contain the used variables.
-        for (LogicalVariable v : usedVariables) {
-            EquivalenceClass ec = eqClasses.get(v);
-            if (ec == null) {
-                EquivalenceClass oc = chldClasses.get(v);
-                if (oc == null) {
-                    continue;
-                }
-                List<LogicalVariable> m = new LinkedList<LogicalVariable>();
-                for (LogicalVariable v2 : oc.getMembers()) {
-                    if (usedVariables.contains(v2)) {
-                        m.add(v2);
-                    }
-                }
-                EquivalenceClass nc;
-                if (oc.representativeIsConst()) {
-                    nc = new EquivalenceClass(m, oc.getConstRepresentative());
-                } else if (m.contains(oc.getVariableRepresentative())) {
-                    nc = new EquivalenceClass(m, oc.getVariableRepresentative());
-                } else {
-                    nc = new EquivalenceClass(m, v);
-                }
-                for (LogicalVariable v3 : m) {
-                    eqClasses.put(v3, nc);
-                }
-            }
-        }
-
-        // Propagates equivalent classes that contain expressions that use the used variables.
-        // Note that for the case variable $v is not in the used variables but it is
-        // equivalent to field-access($t, i) and $t is a used variable, the equivalent
-        // class should still be propagated (kept).
-        Set<LogicalVariable> usedVarSet = new HashSet<LogicalVariable>(usedVariables);
-        for (Entry<LogicalVariable, EquivalenceClass> entry : chldClasses.entrySet()) {
-            EquivalenceClass ec = entry.getValue();
-            for (ILogicalExpression expr : ec.getExpressionMembers()) {
-                Set<LogicalVariable> exprUsedVars = new HashSet<LogicalVariable>();
-                expr.getUsedVariables(exprUsedVars);
-                exprUsedVars.retainAll(usedVarSet);
-                // Check if the expression member uses a used variable.
-                if (!exprUsedVars.isEmpty()) {
-                    for (LogicalVariable v : ec.getMembers()) {
-                        eqClasses.put(v, ec);
-                        // If variable members contain a used variable, the representative
-                        // variable should be a used variable.
-                        if (usedVarSet.contains(v)) {
-                            ec.setVariableRepresentative(v);
-                        }
-                    }
-                }
-            }
-        }
-
-        List<FunctionalDependency> chldFds = getOrComputeFDs(op2, ctx);
-        for (FunctionalDependency fd : chldFds) {
-            if (!usedVariables.containsAll(fd.getHead())) {
-                continue;
-            }
-            List<LogicalVariable> tl = new LinkedList<LogicalVariable>();
-            for (LogicalVariable v : fd.getTail()) {
-                if (usedVariables.contains(v)) {
-                    tl.add(v);
-                }
-            }
-            if (!tl.isEmpty()) {
-                FunctionalDependency newFd = new FunctionalDependency(fd.getHead(), tl);
-                fds.add(newFd);
-            }
-        }
-    }
-
-    private void fdsEqClassesForAbstractUnnestOperator(AbstractUnnestOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
-        Map<LogicalVariable, EquivalenceClass> eqClasses = getOrCreateEqClasses(op, ctx);
-        Map<LogicalVariable, EquivalenceClass> propagatedEqClasses = getOrComputeEqClasses(inp1, ctx);
-        /**
-         * The original eq classes of unnest-map are only for produced variables, therefore
-         * eqClasses and propagatedEqClasses do not have overlaps.
-         */
-        eqClasses.putAll(propagatedEqClasses);
-        ctx.putEquivalenceClassMap(op, eqClasses);
-        List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
-        ctx.putFDList(op, fds);
-
-        ILogicalExpression expr = op.getExpressionRef().getValue();
-        if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            AbstractFunctionCallExpression afe = (AbstractFunctionCallExpression) expr;
-            if (afe.getKind() == FunctionKind.UNNEST && ((UnnestingFunctionCallExpression) afe).returnsUniqueValues()) {
-                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-                VariableUtilities.getLiveVariables(op, vars);
-                ArrayList<LogicalVariable> h = new ArrayList<LogicalVariable>();
-                h.addAll(op.getVariables());
-                FunctionalDependency fd = new FunctionalDependency(h, vars);
-                fds.add(fd);
-            }
-        }
-    }
-
-    public static void setEmptyFDsEqClasses(ILogicalOperator op, IOptimizationContext ctx) {
-        Map<LogicalVariable, EquivalenceClass> eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
-        ctx.putEquivalenceClassMap(op, eqClasses);
-        List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
-        ctx.putFDList(op, fds);
-    }
-
-    private LogicalVariable getNewGbyVar(GroupByOperator g, LogicalVariable v) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getGroupByList()) {
-            ILogicalExpression e = p.second.getValue();
-            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
-                if (v2 == v) {
-                    return p.first;
-                }
-            }
-        }
-        return null;
-    }
-
-    private LogicalVariable getNewDecorVar(GroupByOperator g, LogicalVariable v) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
-            ILogicalExpression e = p.second.getValue();
-            if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
-                if (v2 == v) {
-                    return (p.first != null) ? p.first : v2;
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, IOptimizationContext ctx)
-            throws AlgebricksException {
-        propagateFDsAndEquivClasses(op, ctx);
-        return null;
-    }
-
-    /**
-     * Propagate equivalences that carried in expressions to the variables that
-     * they are assigned to.
-     * 
-     * @param eqClasses
-     *            an equivalent class map
-     * @param assignExprs
-     *            expressions on the right-hand-side of assignments
-     * @param assignVars
-     *            variables on the left-hand-side of assignments
-     */
-    private void propagateEquivalenceFromExpressionsToVariables(Map<LogicalVariable, EquivalenceClass> eqClasses,
-            List<Mutable<ILogicalExpression>> assignExprs, List<LogicalVariable> assignVars) {
-        for (int assignVarIndex = 0; assignVarIndex < assignVars.size(); ++assignVarIndex) {
-            LogicalVariable var = assignVars.get(assignVarIndex);
-            ILogicalExpression expr = assignExprs.get(assignVarIndex).getValue();
-            for (Entry<LogicalVariable, EquivalenceClass> entry : eqClasses.entrySet()) {
-                EquivalenceClass eqc = entry.getValue();
-                // If the equivalence class contains the right-hand-side expression,
-                // the left-hand-side variable is added into the equivalence class.
-                if (eqc.contains(expr)) {
-                    eqc.addMember(var);
-                }
-            }
-        }
-    }
-
-}
\ No newline at end of file

[27/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
new file mode 100644
index 0000000..299f519
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+
+public abstract class AbstractScanPOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
new file mode 100644
index 0000000..4b85c56
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+
+public abstract class AbstractStableSortPOperator extends AbstractPhysicalOperator {
+
+    protected OrderColumn[] sortColumns;
+    protected ILocalStructuralProperty orderProp;
+
+    public AbstractStableSortPOperator() {
+    }
+
+    public OrderColumn[] getSortColumns() {
+        return sortColumns;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        // if (orderProps == null) { // to do caching, we need some mechanism to
+        // invalidate cache
+        computeLocalProperties(op);
+        // }
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        StructuralPropertiesVector childProp = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties();
+        deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(),
+                Collections.singletonList(orderProp));
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop,
+            IPhysicalPropertiesVector reqdByParent) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+            if (orderProp == null) {
+                computeLocalProperties(op);
+            }
+            StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(
+                    IPartitioningProperty.UNPARTITIONED, Collections.singletonList(orderProp)) };
+            return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+        } else {
+            return emptyUnaryRequirements();
+        }
+    }
+
+    public void computeLocalProperties(ILogicalOperator op) {
+        OrderOperator ord = (OrderOperator) op;
+        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+        for (Pair<IOrder, Mutable<ILogicalExpression>> p : ord.getOrderExpressions()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+                LogicalVariable var = varRef.getVariableReference();
+                orderColumns.add(new OrderColumn(var, p.first.getKind()));
+            } else {
+                throw new IllegalStateException();
+            }
+        }
+        sortColumns = orderColumns.toArray(new OrderColumn[orderColumns.size()]);
+        orderProp = new LocalOrderProperty(orderColumns);
+    }
+
+    public ILocalStructuralProperty getOrderProperty() {
+        return orderProp;
+    }
+
+    @Override
+    public String toString() {
+        if (orderProp == null) {
+            return getOperatorTag().toString();
+        } else {
+            return getOperatorTag().toString() + " " + orderProp;
+        }
+    }
+
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[] { 0 };
+        int[] outputDependencyLabels = new int[] { 1 };
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
new file mode 100644
index 0000000..8b1b447
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.AggregateRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class AggregatePOperator extends AbstractPhysicalOperator {
+
+    public AggregatePOperator() {
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.AGGREGATE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AggregateOperator aggOp = (AggregateOperator) op;
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        if (aggOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+            deliveredProperties = new StructuralPropertiesVector(op2.getDeliveredPhysicalProperties()
+                    .getPartitioningProperty(), new ArrayList<ILocalStructuralProperty>());
+        } else {
+            deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
+                    new ArrayList<ILocalStructuralProperty>());
+        }
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        AggregateOperator aggOp = (AggregateOperator) op;
+        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+        if (aggOp.isGlobal() && aggOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
+            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+        } else {
+            return emptyUnaryRequirements();
+        }
+
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        AggregateOperator aggOp = (AggregateOperator) op;
+        List<LogicalVariable> variables = aggOp.getVariables();
+        List<Mutable<ILogicalExpression>> expressions = aggOp.getExpressions();
+        int[] outColumns = new int[variables.size()];
+        for (int i = 0; i < outColumns.length; i++) {
+            outColumns[i] = opSchema.findVariable(variables.get(i));
+        }
+        IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[expressions.size()];
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        for (int i = 0; i < aggFactories.length; i++) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i).getValue();
+            aggFactories[i] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun,
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+        }
+
+        AggregateRuntimeFactory runtime = new AggregateRuntimeFactory(aggFactories);
+
+        // contribute one Asterix framewriter
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        builder.contributeMicroOperator(aggOp, runtime, recDesc);
+        // and contribute one edge from its child
+        ILogicalOperator src = aggOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, aggOp, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
new file mode 100644
index 0000000..0a67ced
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class AssignPOperator extends AbstractPhysicalOperator {
+
+    private boolean flushFramesRapidly;
+    private int cardinalityConstraint = 0;
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.ASSIGN;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AssignOperator assignOp = (AssignOperator) op;
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+        if (assignOp.getExplicitOrderingProperty() != null) {
+            deliveredProperties.getLocalProperties().add(assignOp.getExplicitOrderingProperty());
+        }
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        AssignOperator assign = (AssignOperator) op;
+        List<LogicalVariable> variables = assign.getVariables();
+        List<Mutable<ILogicalExpression>> expressions = assign.getExpressions();
+        int[] outColumns = new int[variables.size()];
+        for (int i = 0; i < outColumns.length; i++) {
+            outColumns[i] = opSchema.findVariable(variables.get(i));
+        }
+        IScalarEvaluatorFactory[] evalFactories = new IScalarEvaluatorFactory[expressions.size()];
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        for (int i = 0; i < evalFactories.length; i++) {
+            evalFactories[i] = expressionRuntimeProvider.createEvaluatorFactory(expressions.get(i).getValue(),
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+        }
+
+        // TODO push projections into the operator
+        int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
+
+        AssignRuntimeFactory runtime = new AssignRuntimeFactory(outColumns, evalFactories, projectionList,
+                flushFramesRapidly);
+
+        // contribute one Asterix framewriter
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        if (cardinalityConstraint > 0) {
+            AlgebricksCountPartitionConstraint countConstraint = new AlgebricksCountPartitionConstraint(
+                    cardinalityConstraint);
+            builder.contributeMicroOperator(assign, runtime, recDesc, countConstraint);
+        } else {
+            builder.contributeMicroOperator(assign, runtime, recDesc);
+        }
+        // and contribute one edge from its child
+        ILogicalOperator src = assign.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, assign, 0);
+
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    public void setRapidFrameFlush(boolean flushFramesRapidly) {
+        this.flushFramesRapidly = flushFramesRapidly;
+    }
+
+    public void setCardinalityConstraint(int cardinality) {
+        this.cardinalityConstraint = cardinality;
+    }
+
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
new file mode 100644
index 0000000..b210d56
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
+
+public class BroadcastPOperator extends AbstractExchangePOperator {
+
+    private INodeDomain domain;
+
+    public BroadcastPOperator(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.BROADCAST_EXCHANGE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        IPartitioningProperty pp = new BroadcastPartitioningProperty(domain);
+        this.deliveredProperties = new StructuralPropertiesVector(pp, op2.getDeliveredPhysicalProperties()
+                .getLocalProperties());
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
new file mode 100644
index 0000000..23652f6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class BulkloadPOperator extends AbstractPhysicalOperator {
+
+    private final LogicalVariable payload;
+    private final List<LogicalVariable> primaryKeys;
+    private final List<LogicalVariable> additionalFilteringKeys;
+    private final IDataSource<?> dataSource;
+
+    public BulkloadPOperator(LogicalVariable payload, List<LogicalVariable> keys,
+           List<LogicalVariable> additionalFilteringKeys, IDataSource<?> dataSource) {
+        this.payload = payload;
+        this.primaryKeys = keys;
+        this.additionalFilteringKeys = additionalFilteringKeys;
+        this.dataSource = dataSource;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.BULKLOAD;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<LogicalVariable> scanVariables = new ArrayList<>();
+        scanVariables.addAll(primaryKeys);
+        scanVariables.add(new LogicalVariable(-1));
+        IPhysicalPropertiesVector physicalProps = dataSource.getPropertiesProvider().computePropertiesVector(
+                scanVariables);
+        StructuralPropertiesVector spv = new StructuralPropertiesVector(physicalProps.getPartitioningProperty(),
+                physicalProps.getLocalProperties());
+        return new PhysicalRequirements(new IPhysicalPropertiesVector[] { spv },
+                IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
+        assert insertDeleteOp.getOperation() == Kind.INSERT;
+        assert insertDeleteOp.isBulkload();
+
+        IMetadataProvider mp = context.getMetadataProvider();
+        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
+        JobSpecification spec = builder.getJobSpec();
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getInsertRuntime(
+                dataSource, propagatedSchema, typeEnv, primaryKeys, payload, additionalFilteringKeys,
+                inputDesc, context, spec, true);
+        builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
+        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
new file mode 100644
index 0000000..f1cb937
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+
+@SuppressWarnings("rawtypes")
+public class DataSourceScanPOperator extends AbstractScanPOperator {
+
+    private IDataSource<?> dataSource;
+    private Object implConfig;
+
+    public DataSourceScanPOperator(IDataSource<?> dataSource) {
+        this.dataSource = dataSource;
+    }
+
+    public void setImplConfig(Object implConfig) {
+        this.implConfig = implConfig;
+    }
+
+    public Object getImplConfig() {
+        return implConfig;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.DATASOURCE_SCAN;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        // partitioning properties
+        DataSourceScanOperator dssOp = (DataSourceScanOperator) op;
+        IDataSourcePropertiesProvider dspp = dataSource.getPropertiesProvider();
+        deliveredProperties = dspp.computePropertiesVector(dssOp.getVariables());
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        DataSourceScanOperator scan = (DataSourceScanOperator) op;
+        IMetadataProvider mp = context.getMetadataProvider();
+        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
+        List<LogicalVariable> vars = scan.getVariables();
+        List<LogicalVariable> projectVars = scan.getProjectVariables();
+
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars,
+                projectVars, scan.isProjectPushed(), scan.getMinFilterVars(), scan.getMaxFilterVars(), opSchema,
+                typeEnv, context, builder.getJobSpec(), implConfig);
+        builder.contributeHyracksOperator(scan, p.first);
+        if (p.second != null) {
+            builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
+        }
+
+        ILogicalOperator srcExchange = scan.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(srcExchange, 0, scan, 0);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java
new file mode 100644
index 0000000..54fdf2c
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/DistributeResultPOperator.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class DistributeResultPOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.DISTRIBUTE_RESULT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        DistributeResultOperator write = (DistributeResultOperator) op;
+        IDataSink sink = write.getDataSink();
+        IPartitioningProperty pp = sink.getPartitioningProperty();
+        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp, null) };
+        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        DistributeResultOperator resultOp = (DistributeResultOperator) op;
+        IMetadataProvider mp = context.getMetadataProvider();
+
+        JobSpecification spec = builder.getJobSpec();
+
+        int[] columns = new int[resultOp.getExpressions().size()];
+        int i = 0;
+        for (Mutable<ILogicalExpression> exprRef : resultOp.getExpressions()) {
+            ILogicalExpression expr = exprRef.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new NotImplementedException("Only writing variable expressions is supported.");
+            }
+            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+            LogicalVariable v = varRef.getVariableReference();
+            columns[i++] = inputSchemas[0].findVariable(v);
+        }
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
+
+        IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
+                context, columns);
+
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getResultHandleRuntime(
+                resultOp.getDataSink(), columns, pf, inputDesc, true, spec);
+
+        builder.contributeHyracksOperator(resultOp, runtimeAndConstraints.first);
+        ILogicalOperator src = resultOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, resultOp, 0);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
new file mode 100644
index 0000000..b714979
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class EmptyTupleSourcePOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.EMPTY_TUPLE_SOURCE;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
+                new LinkedList<ILocalStructuralProperty>());
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return null;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        EmptyTupleSourceRuntimeFactory runtime = new EmptyTupleSourceRuntimeFactory();
+        RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+        builder.contributeMicroOperator(op, runtime, recDesc);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
new file mode 100644
index 0000000..fe438a5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.OperatorSchemaImpl;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SerializableAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.external.ExternalGroupOperatorDescriptor;
+
+public class ExternalGroupByPOperator extends AbstractPhysicalOperator {
+
+    private int tableSize = 0;
+    private int frameLimit = 0;
+    private List<LogicalVariable> columnSet = new ArrayList<LogicalVariable>();
+
+    public ExternalGroupByPOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList, int frameLimit,
+            int tableSize) {
+        this.tableSize = tableSize;
+        this.frameLimit = frameLimit;
+        computeColumnSet(gbyList);
+    }
+
+    public void computeColumnSet(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList) {
+        columnSet.clear();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) expr;
+                columnSet.add(v.getVariableReference());
+            }
+        }
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.EXTERNAL_GROUP_BY;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + columnSet;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    public List<LogicalVariable> getGbyColumns() {
+        return columnSet;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
+
+        GroupByOperator gOp = (GroupByOperator) op;
+        Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
+
+        if (!columnSet.isEmpty()) {
+            propsLocal.add(new LocalGroupingProperty(columnSet));
+        }
+        for (ILogicalPlan p : gOp.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator rOp = r.getValue();
+                propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
+            }
+        }
+
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
+        deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
+        if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
+            StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+            pv[0] = new StructuralPropertiesVector(new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(
+                    columnSet), null), null);
+            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+        } else {
+            return emptyUnaryRequirements();
+        }
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        List<LogicalVariable> gbyCols = getGbyColumns();
+        int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]);
+        GroupByOperator gby = (GroupByOperator) op;
+        int numFds = gby.getDecorList().size();
+        int fdColumns[] = new int[numFds];
+        int j = 0;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new AlgebricksException("pre-sorted group-by expects variable references.");
+            }
+            VariableReferenceExpression v = (VariableReferenceExpression) expr;
+            LogicalVariable decor = v.getVariableReference();
+            fdColumns[j++] = inputSchemas[0].findVariable(decor);
+        }
+
+        if (gby.getNestedPlans().size() != 1) {
+            throw new AlgebricksException(
+                    "External group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        ILogicalPlan p0 = gby.getNestedPlans().get(0);
+        if (p0.getRoots().size() != 1) {
+            throw new AlgebricksException(
+                    "External group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+
+        IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
+        List<Object> intermediateTypes = new ArrayList<Object>();
+        int n = aggOp.getExpressions().size();
+        ICopySerializableAggregateFunctionFactory[] aff = new ICopySerializableAggregateFunctionFactory[n];
+        int i = 0;
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
+        IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
+        for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
+            aff[i++] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv,
+                    inputSchemas, context);
+            intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
+                    context.getMetadataProvider()));
+        }
+
+        int[] keyAndDecFields = new int[keys.length + fdColumns.length];
+        for (i = 0; i < keys.length; ++i) {
+            keyAndDecFields[i] = keys[i];
+        }
+        for (i = 0; i < fdColumns.length; i++) {
+            keyAndDecFields[keys.length + i] = fdColumns[i];
+        }
+
+        List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList())
+            keyAndDecVariables.add(p.first);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList())
+            keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
+
+        for (LogicalVariable var : keyAndDecVariables)
+            aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
+
+        compileSubplans(inputSchemas[0], gby, opSchema, context);
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(gbyCols,
+                aggOpInputEnv, context);
+        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        IBinaryHashFunctionFactory[] hashFunctionFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
+                gbyCols, aggOpInputEnv, context);
+
+        ICopySerializableAggregateFunctionFactory[] merges = new ICopySerializableAggregateFunctionFactory[n];
+        List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+        IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
+        localInputSchemas[0] = new OperatorSchemaImpl();
+        for (i = 0; i < n; i++) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
+                    .get(i).getValue();
+            aggFun.getUsedVariables(usedVars);
+        }
+        i = 0;
+        for (Object type : intermediateTypes) {
+            aggOpInputEnv.setVarType(usedVars.get(i++), type);
+        }
+        for (LogicalVariable keyVar : keyAndDecVariables)
+            localInputSchemas[0].addVariable(keyVar);
+        for (LogicalVariable usedVar : usedVars)
+            localInputSchemas[0].addVariable(usedVar);
+        for (i = 0; i < n; i++) {
+            AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
+                    .get(i).getValue();
+            merges[i] = expressionRuntimeProvider.createSerializableAggregateFunctionFactory(mergeFun, aggOpInputEnv,
+                    localInputSchemas, context);
+        }
+        IAggregatorDescriptorFactory aggregatorFactory = new SerializableAggregatorDescriptorFactory(aff);
+        IAggregatorDescriptorFactory mergeFactory = new SerializableAggregatorDescriptorFactory(merges);
+
+        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
+        INormalizedKeyComputerFactory normalizedKeyFactory = JobGenHelper.variablesToAscNormalizedKeyComputerFactory(
+                gbyCols, aggOpInputEnv, context);
+        ExternalGroupOperatorDescriptor gbyOpDesc = new ExternalGroupOperatorDescriptor(spec, keyAndDecFields,
+                frameLimit, comparatorFactories, normalizedKeyFactory, aggregatorFactory, mergeFactory,
+                recordDescriptor, new HashSpillableTableFactory(tpcf, tableSize), false);
+
+        contributeOpDesc(builder, gby, gbyOpDesc);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[] { 0 };
+        int[] outputDependencyLabels = new int[] { 1 };
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
new file mode 100644
index 0000000..1a91958
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
+
+public class HashPartitionExchangePOperator extends AbstractExchangePOperator {
+
+    private List<LogicalVariable> hashFields;
+    private INodeDomain domain;
+
+    public HashPartitionExchangePOperator(List<LogicalVariable> hashFields, INodeDomain domain) {
+        this.hashFields = hashFields;
+        this.domain = domain;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.HASH_PARTITION_EXCHANGE;
+    }
+
+    public List<LogicalVariable> getHashFields() {
+        return hashFields;
+    }
+
+    public INodeDomain getDomain() {
+        return domain;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(hashFields), domain);
+        this.deliveredProperties = new StructuralPropertiesVector(p, null);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " " + hashFields;
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        int[] keys = new int[hashFields.size()];
+        IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[hashFields.size()];
+        int i = 0;
+        IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider();
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        for (LogicalVariable v : hashFields) {
+            keys[i] = opSchema.findVariable(v);
+            hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v));
+            ++i;
+        }
+        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
+        IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
new file mode 100644
index 0000000..fc4f116
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
@@ -0,0 +1,167 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class HashPartitionMergeExchangePOperator extends AbstractExchangePOperator {
+
+    private final List<OrderColumn> orderColumns;
+    private final List<LogicalVariable> partitionFields;
+    private final INodeDomain domain;
+
+    public HashPartitionMergeExchangePOperator(List<OrderColumn> orderColumns, List<LogicalVariable> partitionFields,
+            INodeDomain domain) {
+        this.orderColumns = orderColumns;
+        this.partitionFields = partitionFields;
+        this.domain = domain;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE;
+    }
+
+    public List<OrderColumn> getOrderExpressions() {
+        return orderColumns;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(partitionFields),
+                domain);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
+        List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
+        for (ILocalStructuralProperty prop : op2Locals) {
+            if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+                locals.add(prop);
+            } else {
+                break;
+            }
+        }
+
+        this.deliveredProperties = new StructuralPropertiesVector(p, locals);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<ILocalStructuralProperty> orderProps = new LinkedList<ILocalStructuralProperty>();
+        List<OrderColumn> columns = new ArrayList<OrderColumn>();
+        for (OrderColumn oc : orderColumns) {
+            LogicalVariable var = oc.getColumn();
+            columns.add(new OrderColumn(var, oc.getOrder()));
+        }
+        orderProps.add(new LocalOrderProperty(columns));
+        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
+                orderProps) };
+        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " MERGE:" + orderColumns + " HASH:" + partitionFields;
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        int[] keys = new int[partitionFields.size()];
+        IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[partitionFields.size()];
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        {
+            int i = 0;
+            IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider();
+            for (LogicalVariable v : partitionFields) {
+                keys[i] = opSchema.findVariable(v);
+                hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v));
+                ++i;
+            }
+        }
+        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
+
+        int n = orderColumns.size();
+        int[] sortFields = new int[n];
+        IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[n];
+
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        INormalizedKeyComputerFactory nkcf = null;
+
+        int j = 0;
+        for (OrderColumn oc : orderColumns) {
+            LogicalVariable var = oc.getColumn();
+            sortFields[j] = opSchema.findVariable(var);
+            Object type = env.getVarType(var);
+            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+            comparatorFactories[j] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
+            if (j == 0 && nkcfProvider != null && type != null) {
+                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, oc.getOrder() == OrderKind.ASC);
+            }
+            j++;
+        }
+
+        IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields,
+                comparatorFactories, nkcf);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+    }
+
+    public List<LogicalVariable> getPartitionFields() {
+        return partitionFields;
+    }
+
+    public List<OrderColumn> getOrderColumns() {
+        return orderColumns;
+    }
+
+}

[42/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
deleted file mode 100644
index f3cf0a4..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class VariableUtilities {
-
-    public static void getUsedVariables(ILogicalOperator op, Collection<LogicalVariable> usedVariables)
-            throws AlgebricksException {
-        ILogicalOperatorVisitor<Void, Void> visitor = new UsedVariableVisitor(usedVariables);
-        op.accept(visitor, null);
-    }
-
-    public static void getProducedVariables(ILogicalOperator op, Collection<LogicalVariable> producedVariables)
-            throws AlgebricksException {
-        ILogicalOperatorVisitor<Void, Void> visitor = new ProducedVariableVisitor(producedVariables);
-        op.accept(visitor, null);
-    }
-
-    public static void getLiveVariables(ILogicalOperator op, Collection<LogicalVariable> schemaVariables)
-            throws AlgebricksException {
-        ILogicalOperatorVisitor<Void, Void> visitor = new SchemaVariableVisitor(schemaVariables);
-        op.accept(visitor, null);
-    }
-
-    public static void getUsedVariablesInDescendantsAndSelf(ILogicalOperator op, Collection<LogicalVariable> vars)
-            throws AlgebricksException {
-        // DFS traversal
-        VariableUtilities.getUsedVariables(op, vars);
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            getUsedVariablesInDescendantsAndSelf(c.getValue(), vars);
-        }
-    }
-
-    public static void getProducedVariablesInDescendantsAndSelf(ILogicalOperator op, Collection<LogicalVariable> vars)
-            throws AlgebricksException {
-        // DFS traversal
-        VariableUtilities.getProducedVariables(op, vars);
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            getProducedVariablesInDescendantsAndSelf(c.getValue(), vars);
-        }
-    }
-
-    public static void substituteVariables(ILogicalOperator op, LogicalVariable v1, LogicalVariable v2,
-            ITypingContext ctx) throws AlgebricksException {
-        substituteVariables(op, v1, v2, true, ctx);
-    }
-
-    public static void substituteVariablesInDescendantsAndSelf(ILogicalOperator op, LogicalVariable v1,
-            LogicalVariable v2, ITypingContext ctx) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> childOp : op.getInputs()) {
-            substituteVariablesInDescendantsAndSelf(childOp.getValue(), v1, v2, ctx);
-        }
-        substituteVariables(op, v1, v2, true, ctx);
-    }
-
-    public static void substituteVariables(ILogicalOperator op, LogicalVariable v1, LogicalVariable v2,
-            boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
-        ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> visitor = new SubstituteVariableVisitor(
-                goThroughNts, ctx);
-        op.accept(visitor, new Pair<LogicalVariable, LogicalVariable>(v1, v2));
-    }
-
-    public static <T> boolean varListEqualUnordered(List<T> var, List<T> varArg) {
-        Set<T> varSet = new HashSet<T>();
-        Set<T> varArgSet = new HashSet<T>();
-        varSet.addAll(var);
-        varArgSet.addAll(varArg);
-        return varSet.equals(varArgSet);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
deleted file mode 100644
index 42d964d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-
-public abstract class AbstractExchangePOperator extends AbstractPhysicalOperator {
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op,
-                opSchema, context);
-        builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second);
-        ILogicalOperator src = op.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, op, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-
-    public abstract Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(
-            IConnectorDescriptorRegistry spec, ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
deleted file mode 100644
index 764159d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-
-public abstract class AbstractHashJoinPOperator extends AbstractJoinPOperator {
-
-    protected List<LogicalVariable> keysLeftBranch;
-    protected List<LogicalVariable> keysRightBranch;
-
-    public AbstractHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
-            List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities) {
-        super(kind, partitioningType);
-        this.keysLeftBranch = sideLeftOfEqualities;
-        this.keysRightBranch = sideRightOfEqualities;
-    }
-
-    public List<LogicalVariable> getKeysLeftBranch() {
-        return keysLeftBranch;
-    }
-
-    public List<LogicalVariable> getKeysRightBranch() {
-        return keysRightBranch;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context)
-            throws AlgebricksException {
-        IPartitioningProperty pp;
-        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
-
-        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
-            AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-            IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
-            AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
-            IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties();
-
-            if (pv0 == null || pv1 == null) {
-                pp = null;
-            } else {
-                pp = pv0.getPartitioningProperty();
-            }
-        } else {
-            pp = IPartitioningProperty.UNPARTITIONED;
-        }
-        this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context));
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop,
-            IPhysicalPropertiesVector reqdByParent) {
-        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
-        // In a cost-based optimizer, we would also try to propagate the
-        // parent's partitioning requirements.
-        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
-
-        IPartitioningProperty pp1 = null;
-        IPartitioningProperty pp2 = null;
-        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
-            switch (partitioningType) {
-                case PAIRWISE: {
-                    pp1 = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(keysLeftBranch), null);
-                    pp2 = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(keysRightBranch), null);
-                    break;
-                }
-                case BROADCAST: {
-                    pp2 = new BroadcastPartitioningProperty(null);
-                    break;
-                }
-                default: {
-                    throw new IllegalStateException();
-                }
-            }
-        }
-
-        pv[0] = new StructuralPropertiesVector(pp1, null);
-        pv[1] = new StructuralPropertiesVector(pp2, null);
-
-        IPartitioningRequirementsCoordinator prc;
-        switch (kind) {
-            case INNER: {
-                prc = IPartitioningRequirementsCoordinator.EQCLASS_PARTITIONING_COORDINATOR;
-                break;
-            }
-            case LEFT_OUTER: {
-                prc = new IPartitioningRequirementsCoordinator() {
-
-                    @Override
-                    public Pair<Boolean, IPartitioningProperty> coordinateRequirements(
-                            IPartitioningProperty requirements, IPartitioningProperty firstDeliveredPartitioning,
-                            ILogicalOperator op, IOptimizationContext context) throws AlgebricksException {
-                        if (firstDeliveredPartitioning != null
-                                && firstDeliveredPartitioning.getPartitioningType() == requirements
-                                        .getPartitioningType()) {
-                            switch (requirements.getPartitioningType()) {
-                                case UNORDERED_PARTITIONED: {
-                                    UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
-                                    Set<LogicalVariable> set1 = upp1.getColumnSet();
-                                    UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) requirements;
-                                    Set<LogicalVariable> modifuppreq = new ListSet<LogicalVariable>();
-                                    Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
-                                    Set<LogicalVariable> covered = new ListSet<LogicalVariable>();
-                                    for (LogicalVariable r : uppreq.getColumnSet()) {
-                                        EquivalenceClass ecSnd = eqmap.get(r);
-                                        boolean found = false;
-                                        int j = 0;
-                                        for (LogicalVariable rvar : keysRightBranch) {
-                                            if (rvar == r || ecSnd != null && eqmap.get(rvar) == ecSnd) {
-                                                found = true;
-                                                break;
-                                            }
-                                            j++;
-                                        }
-                                        if (!found) {
-                                            throw new IllegalStateException("Did not find a variable equivalent to "
-                                                    + r + " among " + keysRightBranch);
-                                        }
-                                        LogicalVariable v2 = keysLeftBranch.get(j);
-                                        EquivalenceClass ecFst = eqmap.get(v2);
-                                        for (LogicalVariable vset1 : set1) {
-                                            if (vset1 == v2 || ecFst != null && eqmap.get(vset1) == ecFst) {
-                                                covered.add(vset1);
-                                                modifuppreq.add(r);
-                                                break;
-                                            }
-                                        }
-                                    }
-                                    if (!covered.equals(set1)) {
-                                        throw new AlgebricksException("Could not modify " + requirements
-                                                + " to agree with partitioning property " + firstDeliveredPartitioning
-                                                + " delivered by previous input operator.");
-                                    }
-                                    UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq,
-                                            requirements.getNodeDomain());
-                                    return new Pair<Boolean, IPartitioningProperty>(false, upp2);
-                                }
-                                case ORDERED_PARTITIONED: {
-                                    throw new NotImplementedException();
-                                }
-                            }
-                        }
-                        return new Pair<Boolean, IPartitioningProperty>(true, requirements);
-                    }
-                };
-                break;
-            }
-            default: {
-                throw new IllegalStateException();
-            }
-        }
-
-        return new PhysicalRequirements(pv, prc);
-    }
-
-    protected abstract List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op,
-            IOptimizationContext context) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
deleted file mode 100644
index fcc04ab..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
-
-public abstract class AbstractJoinPOperator extends AbstractPhysicalOperator {
-
-    public enum JoinPartitioningType {
-        PAIRWISE, BROADCAST
-    }
-
-    protected final JoinKind kind;
-    protected final JoinPartitioningType partitioningType;
-
-    public AbstractJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType) {
-        this.kind = kind;
-        this.partitioningType = partitioningType;
-    }
-
-    public JoinKind getKind() {
-        return kind;
-    }
-
-    public JoinPartitioningType getPartitioningType() {
-        return partitioningType;
-    }
-
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[] { 1, 0 };
-        int[] outputDependencyLabels = new int[] { 1 };
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
deleted file mode 100644
index 66e7b98..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.PlanCompiler;
-import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public abstract class AbstractPhysicalOperator implements IPhysicalOperator {
-
-    protected IPhysicalPropertiesVector deliveredProperties;
-    private boolean disableJobGenBelow = false;
-    private Object hostQueryContext;
-
-    @Override
-    public final IPhysicalPropertiesVector getDeliveredProperties() {
-        return deliveredProperties;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString();
-    }
-
-    public void setHostQueryContext(Object context) {
-        this.hostQueryContext = context;
-    }
-
-    public Object getHostQueryContext() {
-        return hostQueryContext;
-    }
-
-    protected PhysicalRequirements emptyUnaryRequirements() {
-        StructuralPropertiesVector[] req = new StructuralPropertiesVector[] { StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR };
-        return new PhysicalRequirements(req, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    protected PhysicalRequirements emptyUnaryRequirements(int numberOfChildren) {
-        StructuralPropertiesVector[] req = new StructuralPropertiesVector[numberOfChildren];
-        for (int i = 0; i < numberOfChildren; i++) {
-            req[i] = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
-        }
-        return new PhysicalRequirements(req, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void disableJobGenBelowMe() {
-        this.disableJobGenBelow = true;
-    }
-
-    @Override
-    public boolean isJobGenDisabledBelowMe() {
-        return disableJobGenBelow;
-    }
-
-    /**
-     * @return labels (0 or 1) for each input and output indicating the dependency between them.
-     *         The edges labeled as 1 must wait for the edges with label 0.
-     */
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[op.getInputs().size()]; // filled with 0's
-        int[] outputDependencyLabels = new int[] { 0 };
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    protected void contributeOpDesc(IHyracksJobBuilder builder, AbstractLogicalOperator op, IOperatorDescriptor opDesc) {
-        if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
-            AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1);
-            builder.contributeAlgebricksPartitionConstraint(opDesc, apc);
-        }
-        builder.contributeHyracksOperator(op, opDesc);
-    }
-
-    protected AlgebricksPipeline[] compileSubplans(IOperatorSchema outerPlanSchema,
-            AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, JobGenContext context)
-            throws AlgebricksException {
-        AlgebricksPipeline[] subplans = new AlgebricksPipeline[npOp.getNestedPlans().size()];
-        PlanCompiler pc = new PlanCompiler(context);
-        int i = 0;
-        for (ILogicalPlan p : npOp.getNestedPlans()) {
-            subplans[i++] = buildPipelineWithProjection(p, outerPlanSchema, npOp, opSchema, pc);
-        }
-        return subplans;
-    }
-
-    private AlgebricksPipeline buildPipelineWithProjection(ILogicalPlan p, IOperatorSchema outerPlanSchema,
-            AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, PlanCompiler pc) throws AlgebricksException {
-        if (p.getRoots().size() > 1) {
-            throw new NotImplementedException("Nested plans with several roots are not supported.");
-        }
-        JobSpecification nestedJob = pc.compilePlan(p, outerPlanSchema, null);
-        ILogicalOperator topOpInSubplan = p.getRoots().get(0).getValue();
-        JobGenContext context = pc.getContext();
-        IOperatorSchema topOpInSubplanScm = context.getSchema(topOpInSubplan);
-        opSchema.addAllVariables(topOpInSubplanScm);
-
-        Map<OperatorDescriptorId, IOperatorDescriptor> opMap = nestedJob.getOperatorMap();
-        if (opMap.size() != 1) {
-            throw new AlgebricksException(
-                    "Attempting to construct a nested plan with "
-                            + opMap.size()
-                            + " operator descriptors. Currently, nested plans can only consist in linear pipelines of Asterix micro operators.");
-        }
-
-        for (OperatorDescriptorId oid : opMap.keySet()) {
-            IOperatorDescriptor opd = opMap.get(oid);
-            if (!(opd instanceof AlgebricksMetaOperatorDescriptor)) {
-                throw new AlgebricksException(
-                        "Can only generate Hyracks jobs for pipelinable Asterix nested plans, not for "
-                                + opd.getClass().getName());
-            }
-            AlgebricksMetaOperatorDescriptor amod = (AlgebricksMetaOperatorDescriptor) opd;
-
-            return amod.getPipeline();
-            // we suppose that the top operator in the subplan already does the
-            // projection for us
-        }
-
-        throw new IllegalStateException();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
deleted file mode 100644
index 41711cb..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-
-public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysicalOperator {
-
-    protected List<LogicalVariable> columnList;
-
-    public AbstractPreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
-        this.columnList = columnList;
-    }
-
-    @Override
-    public String toString() {
-        return getOperatorTag().toString() + columnList;
-    }
-
-    public List<LogicalVariable> getGbyColumns() {
-        return columnList;
-    }
-
-    public void setGbyColumns(List<LogicalVariable> gByColumns) {
-        this.columnList = gByColumns;
-    }
-
-    // Obs: We don't propagate properties corresponding to decors, since they
-    // are func. dep. on the group-by variables.
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
-        GroupByOperator gby = (GroupByOperator) op;
-        ILogicalOperator op2 = gby.getInputs().get(0).getValue();
-        IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
-        IPartitioningProperty pp = childProp.getPartitioningProperty();
-        List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties();
-        if (childLocals != null) {
-            for (ILocalStructuralProperty lsp : childLocals) {
-                boolean failed = false;
-                switch (lsp.getPropertyType()) {
-                    case LOCAL_GROUPING_PROPERTY: {
-                        LocalGroupingProperty lgp = (LocalGroupingProperty) lsp;
-                        Set<LogicalVariable> colSet = new ListSet<LogicalVariable>();
-                        for (LogicalVariable v : lgp.getColumnSet()) {
-                            LogicalVariable v2 = getLhsGbyVar(gby, v);
-                            if (v2 != null) {
-                                colSet.add(v2);
-                            } else {
-                                failed = true;
-                            }
-                        }
-                        if (!failed) {
-                            propsLocal.add(new LocalGroupingProperty(colSet));
-                        }
-                        break;
-                    }
-                    case LOCAL_ORDER_PROPERTY: {
-                        LocalOrderProperty lop = (LocalOrderProperty) lsp;
-                        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-                        for (OrderColumn oc : lop.getOrderColumns()) {
-                            LogicalVariable v2 = getLhsGbyVar(gby, oc.getColumn());
-                            if (v2 != null) {
-                                orderColumns.add(new OrderColumn(v2, oc.getOrder()));
-                            } else {
-                                failed = true;
-                            }
-                        }
-                        if (!failed) {
-                            propsLocal.add(new LocalOrderProperty(orderColumns));
-                        }
-                        break;
-                    }
-                    default: {
-                        throw new IllegalStateException();
-                    }
-                }
-                if (failed) {
-                    break;
-                }
-            }
-        }
-        deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
-        List<ILocalStructuralProperty> localProps = null;
-
-        localProps = new ArrayList<ILocalStructuralProperty>(1);
-        Set<LogicalVariable> gbvars = new ListSet<LogicalVariable>(columnList);
-        LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<LogicalVariable>(columnList));
-
-        GroupByOperator gby = (GroupByOperator) op;
-        boolean goon = true;
-        for (ILogicalPlan p : gby.getNestedPlans()) {
-            // try to propagate secondary order requirements from nested
-            // groupings
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
-                if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
-                    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-                    IPhysicalOperator pop2 = op2.getPhysicalOperator();
-                    if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
-                        List<LogicalVariable> sndOrder = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
-                        groupProp.getColumnSet().addAll(sndOrder);
-                        groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
-                        goon = false;
-                        break;
-                    }
-                }
-            }
-            if (!goon) {
-                break;
-            }
-        }
-
-        localProps.add(groupProp);
-
-        if (reqdByParent != null) {
-            // propagate parent requirements
-            List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
-            if (lpPar != null) {
-                boolean allOk = true;
-                List<ILocalStructuralProperty> props = new ArrayList<ILocalStructuralProperty>(lpPar.size());
-                for (ILocalStructuralProperty prop : lpPar) {
-                    if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
-                        allOk = false;
-                        break;
-                    }
-                    LocalOrderProperty lop = (LocalOrderProperty) prop;
-                    List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-                    List<OrderColumn> ords = lop.getOrderColumns();
-                    for (OrderColumn ord : ords) {
-                        Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord.getColumn());
-                        if (p == null) {
-                            p = getDecorPairByRhsVar(gby, ord.getColumn());
-                            if (p == null) {
-                                allOk = false;
-                                break;
-                            }
-                        }
-                        ILogicalExpression e = p.second.getValue();
-                        if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                            throw new IllegalStateException(
-                                    "Right hand side of group-by assignment should have been normalized to a variable reference.");
-                        }
-                        LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
-                        orderColumns.add(new OrderColumn(v, ord.getOrder()));
-                    }
-                    props.add(new LocalOrderProperty(orderColumns));
-                }
-                List<FunctionalDependency> fdList = new ArrayList<FunctionalDependency>();
-                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
-                    List<LogicalVariable> hd = gby.getGbyVarList();
-                    List<LogicalVariable> tl = new ArrayList<LogicalVariable>(1);
-                    tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
-                    fdList.add(new FunctionalDependency(hd, tl));
-                }
-                if (allOk
-                        && PropertiesUtil.matchLocalProperties(localProps, props,
-                                new HashMap<LogicalVariable, EquivalenceClass>(), fdList)) {
-                    localProps = props;
-                }
-            }
-        }
-
-        IPartitioningProperty pp = null;
-        AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
-        if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
-            pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), null);
-        }
-        pv[0] = new StructuralPropertiesVector(pp, localProps);
-        return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getGbyPairByRhsVar(GroupByOperator gby,
-            LogicalVariable var) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
-            if (ve.first == var) {
-                return ve;
-            }
-        }
-        return null;
-    }
-
-    private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getDecorPairByRhsVar(GroupByOperator gby,
-            LogicalVariable var) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getDecorList()) {
-            if (ve.first == var) {
-                return ve;
-            }
-        }
-        return null;
-    }
-
-    private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable var) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
-            ILogicalExpression e = ve.second.getValue();
-            if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-                throw new IllegalStateException(
-                        "Right hand side of group by assignment should have been normalized to a variable reference.");
-            }
-            LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
-            if (v == var) {
-                return ve.first;
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
deleted file mode 100644
index c77222b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-
-public abstract class AbstractPropagatePropertiesForUsedVariablesPOperator extends AbstractPhysicalOperator {
-
-    public void computeDeliveredPropertiesForUsedVariables(ILogicalOperator op, List<LogicalVariable> usedVariables) {
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
-        List<ILocalStructuralProperty> downPropsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
-        List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
-        for (ILocalStructuralProperty lsp : downPropsLocal) {
-            LinkedList<LogicalVariable> cols = new LinkedList<LogicalVariable>();
-            lsp.getColumns(cols);
-            ILocalStructuralProperty propagatedProp = lsp.retainVariables(usedVariables);
-            if (propagatedProp != null) {
-                propsLocal.add(propagatedProp);
-            }
-        }
-        deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
deleted file mode 100644
index 299f519..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-
-public abstract class AbstractScanPOperator extends AbstractPhysicalOperator {
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
deleted file mode 100644
index 4b85c56..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-
-public abstract class AbstractStableSortPOperator extends AbstractPhysicalOperator {
-
-    protected OrderColumn[] sortColumns;
-    protected ILocalStructuralProperty orderProp;
-
-    public AbstractStableSortPOperator() {
-    }
-
-    public OrderColumn[] getSortColumns() {
-        return sortColumns;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        // if (orderProps == null) { // to do caching, we need some mechanism to
-        // invalidate cache
-        computeLocalProperties(op);
-        // }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        StructuralPropertiesVector childProp = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties();
-        deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(),
-                Collections.singletonList(orderProp));
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop,
-            IPhysicalPropertiesVector reqdByParent) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
-        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
-            if (orderProp == null) {
-                computeLocalProperties(op);
-            }
-            StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(
-                    IPartitioningProperty.UNPARTITIONED, Collections.singletonList(orderProp)) };
-            return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-        } else {
-            return emptyUnaryRequirements();
-        }
-    }
-
-    public void computeLocalProperties(ILogicalOperator op) {
-        OrderOperator ord = (OrderOperator) op;
-        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-        for (Pair<IOrder, Mutable<ILogicalExpression>> p : ord.getOrderExpressions()) {
-            ILogicalExpression expr = p.second.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
-                LogicalVariable var = varRef.getVariableReference();
-                orderColumns.add(new OrderColumn(var, p.first.getKind()));
-            } else {
-                throw new IllegalStateException();
-            }
-        }
-        sortColumns = orderColumns.toArray(new OrderColumn[orderColumns.size()]);
-        orderProp = new LocalOrderProperty(orderColumns);
-    }
-
-    public ILocalStructuralProperty getOrderProperty() {
-        return orderProp;
-    }
-
-    @Override
-    public String toString() {
-        if (orderProp == null) {
-            return getOperatorTag().toString();
-        } else {
-            return getOperatorTag().toString() + " " + orderProp;
-        }
-    }
-
-    @Override
-    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
-        int[] inputDependencyLabels = new int[] { 0 };
-        int[] outputDependencyLabels = new int[] { 1 };
-        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
deleted file mode 100644
index 8b1b447..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.AggregateRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class AggregatePOperator extends AbstractPhysicalOperator {
-
-    public AggregatePOperator() {
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.AGGREGATE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AggregateOperator aggOp = (AggregateOperator) op;
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        if (aggOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
-            deliveredProperties = new StructuralPropertiesVector(op2.getDeliveredPhysicalProperties()
-                    .getPartitioningProperty(), new ArrayList<ILocalStructuralProperty>());
-        } else {
-            deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
-                    new ArrayList<ILocalStructuralProperty>());
-        }
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        AggregateOperator aggOp = (AggregateOperator) op;
-        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
-        if (aggOp.isGlobal() && aggOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
-            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
-            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
-        } else {
-            return emptyUnaryRequirements();
-        }
-
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        AggregateOperator aggOp = (AggregateOperator) op;
-        List<LogicalVariable> variables = aggOp.getVariables();
-        List<Mutable<ILogicalExpression>> expressions = aggOp.getExpressions();
-        int[] outColumns = new int[variables.size()];
-        for (int i = 0; i < outColumns.length; i++) {
-            outColumns[i] = opSchema.findVariable(variables.get(i));
-        }
-        IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[expressions.size()];
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        for (int i = 0; i < aggFactories.length; i++) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i).getValue();
-            aggFactories[i] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun,
-                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
-        }
-
-        AggregateRuntimeFactory runtime = new AggregateRuntimeFactory(aggFactories);
-
-        // contribute one Asterix framewriter
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        builder.contributeMicroOperator(aggOp, runtime, recDesc);
-        // and contribute one edge from its child
-        ILogicalOperator src = aggOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, aggOp, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
deleted file mode 100644
index 0a67ced..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksCountPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.AssignRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class AssignPOperator extends AbstractPhysicalOperator {
-
-    private boolean flushFramesRapidly;
-    private int cardinalityConstraint = 0;
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.ASSIGN;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AssignOperator assignOp = (AssignOperator) op;
-        ILogicalOperator op2 = op.getInputs().get(0).getValue();
-        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
-        if (assignOp.getExplicitOrderingProperty() != null) {
-            deliveredProperties.getLocalProperties().add(assignOp.getExplicitOrderingProperty());
-        }
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        AssignOperator assign = (AssignOperator) op;
-        List<LogicalVariable> variables = assign.getVariables();
-        List<Mutable<ILogicalExpression>> expressions = assign.getExpressions();
-        int[] outColumns = new int[variables.size()];
-        for (int i = 0; i < outColumns.length; i++) {
-            outColumns[i] = opSchema.findVariable(variables.get(i));
-        }
-        IScalarEvaluatorFactory[] evalFactories = new IScalarEvaluatorFactory[expressions.size()];
-        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
-        for (int i = 0; i < evalFactories.length; i++) {
-            evalFactories[i] = expressionRuntimeProvider.createEvaluatorFactory(expressions.get(i).getValue(),
-                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
-        }
-
-        // TODO push projections into the operator
-        int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
-
-        AssignRuntimeFactory runtime = new AssignRuntimeFactory(outColumns, evalFactories, projectionList,
-                flushFramesRapidly);
-
-        // contribute one Asterix framewriter
-        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
-        if (cardinalityConstraint > 0) {
-            AlgebricksCountPartitionConstraint countConstraint = new AlgebricksCountPartitionConstraint(
-                    cardinalityConstraint);
-            builder.contributeMicroOperator(assign, runtime, recDesc, countConstraint);
-        } else {
-            builder.contributeMicroOperator(assign, runtime, recDesc);
-        }
-        // and contribute one edge from its child
-        ILogicalOperator src = assign.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, assign, 0);
-
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return true;
-    }
-
-    public void setRapidFrameFlush(boolean flushFramesRapidly) {
-        this.flushFramesRapidly = flushFramesRapidly;
-    }
-
-    public void setCardinalityConstraint(int cardinality) {
-        this.cardinalityConstraint = cardinality;
-    }
-
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
deleted file mode 100644
index b210d56..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
-
-public class BroadcastPOperator extends AbstractExchangePOperator {
-
-    private INodeDomain domain;
-
-    public BroadcastPOperator(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.BROADCAST_EXCHANGE;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        IPartitioningProperty pp = new BroadcastPartitioningProperty(domain);
-        this.deliveredProperties = new StructuralPropertiesVector(pp, op2.getDeliveredPhysicalProperties()
-                .getLocalProperties());
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        return emptyUnaryRequirements();
-    }
-
-    @Override
-    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
-            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
-        IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
-        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
deleted file mode 100644
index 23652f6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BulkloadPOperator.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class BulkloadPOperator extends AbstractPhysicalOperator {
-
-    private final LogicalVariable payload;
-    private final List<LogicalVariable> primaryKeys;
-    private final List<LogicalVariable> additionalFilteringKeys;
-    private final IDataSource<?> dataSource;
-
-    public BulkloadPOperator(LogicalVariable payload, List<LogicalVariable> keys,
-           List<LogicalVariable> additionalFilteringKeys, IDataSource<?> dataSource) {
-        this.payload = payload;
-        this.primaryKeys = keys;
-        this.additionalFilteringKeys = additionalFilteringKeys;
-        this.dataSource = dataSource;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.BULKLOAD;
-    }
-
-    @Override
-    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
-            IPhysicalPropertiesVector reqdByParent) {
-        List<LogicalVariable> scanVariables = new ArrayList<>();
-        scanVariables.addAll(primaryKeys);
-        scanVariables.add(new LogicalVariable(-1));
-        IPhysicalPropertiesVector physicalProps = dataSource.getPropertiesProvider().computePropertiesVector(
-                scanVariables);
-        StructuralPropertiesVector spv = new StructuralPropertiesVector(physicalProps.getPartitioningProperty(),
-                physicalProps.getLocalProperties());
-        return new PhysicalRequirements(new IPhysicalPropertiesVector[] { spv },
-                IPartitioningRequirementsCoordinator.NO_COORDINATION);
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
-    }
-
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
-        assert insertDeleteOp.getOperation() == Kind.INSERT;
-        assert insertDeleteOp.isBulkload();
-
-        IMetadataProvider mp = context.getMetadataProvider();
-        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
-        JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
-                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getInsertRuntime(
-                dataSource, propagatedSchema, typeEnv, primaryKeys, payload, additionalFilteringKeys,
-                inputDesc, context, spec, true);
-        builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
-        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public boolean expensiveThanMaterialization() {
-        return false;
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
deleted file mode 100644
index f1cb937..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-
-@SuppressWarnings("rawtypes")
-public class DataSourceScanPOperator extends AbstractScanPOperator {
-
-    private IDataSource<?> dataSource;
-    private Object implConfig;
-
-    public DataSourceScanPOperator(IDataSource<?> dataSource) {
-        this.dataSource = dataSource;
-    }
-
-    public void setImplConfig(Object implConfig) {
-        this.implConfig = implConfig;
-    }
-
-    public Object getImplConfig() {
-        return implConfig;
-    }
-
-    @Override
-    public PhysicalOperatorTag getOperatorTag() {
-        return PhysicalOperatorTag.DATASOURCE_SCAN;
-    }
-
-    @Override
-    public boolean isMicroOperator() {
-        return false;
-    }
-
-    @Override
-    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        // partitioning properties
-        DataSourceScanOperator dssOp = (DataSourceScanOperator) op;
-        IDataSourcePropertiesProvider dspp = dataSource.getPropertiesProvider();
-        deliveredProperties = dspp.computePropertiesVector(dssOp.getVariables());
-    }
-
-    @SuppressWarnings("unchecked")
-    @Override
-    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
-            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
-            throws AlgebricksException {
-        DataSourceScanOperator scan = (DataSourceScanOperator) op;
-        IMetadataProvider mp = context.getMetadataProvider();
-        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
-        List<LogicalVariable> vars = scan.getVariables();
-        List<LogicalVariable> projectVars = scan.getProjectVariables();
-
-        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars,
-                projectVars, scan.isProjectPushed(), scan.getMinFilterVars(), scan.getMaxFilterVars(), opSchema,
-                typeEnv, context, builder.getJobSpec(), implConfig);
-        builder.contributeHyracksOperator(scan, p.first);
-        if (p.second != null) {
-            builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
-        }
-
-        ILogicalOperator srcExchange = scan.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(srcExchange, 0, scan, 0);
-    }
-}

[22/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
new file mode 100644
index 0000000..7290319
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class StructuralPropertiesVector implements IPhysicalPropertiesVector {
+    private List<ILocalStructuralProperty> propsLocal;
+    private IPartitioningProperty propPartitioning;
+
+    public static final StructuralPropertiesVector EMPTY_PROPERTIES_VECTOR = new StructuralPropertiesVector(null,
+            new ArrayList<ILocalStructuralProperty>());
+
+    public StructuralPropertiesVector(IPartitioningProperty propPartitioning, List<ILocalStructuralProperty> propsLocal) {
+        this.propPartitioning = propPartitioning;
+        this.propsLocal = propsLocal;
+    }
+
+    @Override
+    public String toString() {
+        return "propsLocal=" + propsLocal + "\tpropPartioning=" + propPartitioning;
+    }
+
+    @Override
+    public IPartitioningProperty getPartitioningProperty() {
+        return propPartitioning;
+    }
+
+    @Override
+    public List<ILocalStructuralProperty> getLocalProperties() {
+        return propsLocal;
+    }
+
+    @Override
+    public IPhysicalPropertiesVector clone() {
+        List<ILocalStructuralProperty> propsCopy = new LinkedList<ILocalStructuralProperty>();
+        if (propsLocal != null) {
+            propsCopy.addAll(propsLocal);
+        }
+        return new StructuralPropertiesVector(propPartitioning, propsCopy);
+    }
+
+    /**
+     * 
+     * @param reqd
+     *            vector of required properties
+     * @return a vector of properties from pvector that are not delivered by the
+     *         current vector or null if none
+     */
+    @Override
+    public IPhysicalPropertiesVector getUnsatisfiedPropertiesFrom(IPhysicalPropertiesVector reqd,
+            boolean mayExpandProperties, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+            List<FunctionalDependency> fds) {
+        List<ILocalStructuralProperty> plist = reqd.getLocalProperties();
+        List<ILocalStructuralProperty> diffLocals = null;
+        if (plist != null && !plist.isEmpty()) {
+            if (!PropertiesUtil.matchLocalProperties(plist, propsLocal, equivalenceClasses, fds)) {
+                diffLocals = plist;
+            }
+        }
+
+        IPartitioningProperty diffPart = null;
+        IPartitioningProperty reqdPart = reqd.getPartitioningProperty();
+        if (reqdPart != null) {
+            if (mayExpandProperties) {
+                reqdPart.normalize(equivalenceClasses, fds);
+            } else {
+                reqdPart.normalize(equivalenceClasses, null);
+            }
+            propPartitioning.normalize(equivalenceClasses, fds);
+            if (!PropertiesUtil.matchPartitioningProps(reqdPart, propPartitioning, mayExpandProperties)) {
+                diffPart = reqdPart;
+            }
+        }
+
+        if (diffLocals == null && diffPart == null) {
+            return null;
+        } else {
+            return new StructuralPropertiesVector(diffPart, diffLocals);
+        }
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
new file mode 100644
index 0000000..e27edc5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+
+public abstract class TypePropagationPolicy {
+    public static final TypePropagationPolicy ALL = new TypePropagationPolicy() {
+
+        @Override
+        public Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
+                List<LogicalVariable> nonNullVariableList, List<List<LogicalVariable>> correlatedNullableVariableLists,
+                ITypeEnvPointer... typeEnvs) throws AlgebricksException {
+            for (ITypeEnvPointer p : typeEnvs) {
+                IVariableTypeEnvironment env = p.getTypeEnv();
+                if (env == null) {
+                    throw new AlgebricksException("Null environment for pointer " + p + " in getVarType for var=" + var);
+                }
+                Object t = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists);
+                if (t != null) {
+                    if (ntc != null && ntc.canBeNull(t)) {
+                        for (List<LogicalVariable> list : correlatedNullableVariableLists) {
+                            if (list.contains(var)) {
+                                for (LogicalVariable v : list) {
+                                    if (nonNullVariableList.contains(v)) {
+                                        return ntc.getNonOptionalType(t);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    return t;
+                }
+            }
+            return null;
+        }
+    };
+
+    public static final TypePropagationPolicy LEFT_OUTER = new TypePropagationPolicy() {
+
+        @Override
+        public Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
+                List<LogicalVariable> nonNullVariableList, List<List<LogicalVariable>> correlatedNullableVariableLists,
+                ITypeEnvPointer... typeEnvs) throws AlgebricksException {
+            int n = typeEnvs.length;
+            for (int i = 0; i < n; i++) {
+                Object t = typeEnvs[i].getTypeEnv().getVarType(var, nonNullVariableList,
+                        correlatedNullableVariableLists);
+                if (t != null) {
+                    if (i == 0) { // inner branch
+                        return t;
+                    } else { // outer branch
+                        boolean nonNullVarIsProduced = false;
+                        for (LogicalVariable v : nonNullVariableList) {
+                            if (v == var) {
+                                nonNullVarIsProduced = true;
+                                break;
+                            }
+                            if (typeEnvs[i].getTypeEnv().getVarType(v) != null) {
+                                nonNullVarIsProduced = true;
+                                break;
+                            }
+                        }
+                        if (nonNullVarIsProduced) {
+                            return t;
+                        } else {
+                            return ntc.makeNullableType(t);
+                        }
+                    }
+                }
+            }
+            return null;
+        }
+    };
+
+    public abstract Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
+            List<LogicalVariable> nonNullVariableList, List<List<LogicalVariable>> correlatedNullableVariableLists,
+            ITypeEnvPointer... typeEnvs) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
new file mode 100644
index 0000000..3e79192
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public final class UnorderedPartitionedProperty extends AbstractGroupingProperty implements IPartitioningProperty {
+
+    private INodeDomain domain;
+
+    public UnorderedPartitionedProperty(Set<LogicalVariable> partitioningVariables, INodeDomain domain) {
+        super(partitioningVariables);
+        this.domain = domain;
+    }
+
+    @Override
+    public PartitioningType getPartitioningType() {
+        return PartitioningType.UNORDERED_PARTITIONED;
+    }
+
+    @Override
+    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+        normalizeGroupingColumns(equivalenceClasses, fds);
+    }
+
+    @Override
+    public String toString() {
+        return getPartitioningType().toString() + columnSet;
+    }
+
+    @Override
+    public void getColumns(Collection<LogicalVariable> columns) {
+        columns.addAll(columnSet);
+    }
+
+    @Override
+    public INodeDomain getNodeDomain() {
+        return domain;
+    }
+
+    @Override
+    public void setNodeDomain(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
new file mode 100644
index 0000000..7a47c1e
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
+public class UnpartitionedPropertyComputer implements IPropertiesComputer {
+
+    public static final UnpartitionedPropertyComputer INSTANCE = new UnpartitionedPropertyComputer();
+
+    private UnpartitionedPropertyComputer() {
+    }
+
+    @Override
+    public IPartitioningProperty computePartitioningProperty(ILogicalExpression expr) {
+        return IPartitioningProperty.UNPARTITIONED;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
new file mode 100644
index 0000000..5ac3a93
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+
+public abstract class VariablePropagationPolicy {
+    public static final VariablePropagationPolicy ALL = new VariablePropagationPolicy() {
+        @Override
+        public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+            int n = sources.length;
+            for (int i = 0; i < n; i++) {
+                target.addAllNewVariables(sources[i]);
+            }
+        }
+    };
+
+    public static final VariablePropagationPolicy NONE = new VariablePropagationPolicy() {
+        @Override
+        public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+            // do nothing
+        }
+    };
+
+    /**
+     * Adds, from each source, only variables that do not already appear in the
+     * target.
+     * 
+     * 
+     */
+    public static final VariablePropagationPolicy ADDNEWVARIABLES = new VariablePropagationPolicy() {
+        @Override
+        public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+            for (IOperatorSchema s : sources) {
+                for (LogicalVariable v : s) {
+                    if (target.findVariable(v) < 0) {
+                        target.addVariable(v);
+                    }
+                }
+            }
+        }
+    };
+
+    public abstract void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+            throws AlgebricksException;
+
+    public static VariablePropagationPolicy concat(final VariablePropagationPolicy... policies) {
+        return new VariablePropagationPolicy() {
+            @Override
+            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+                    throws AlgebricksException {
+                if (policies.length != sources.length) {
+                    throw new IllegalArgumentException();
+                }
+                for (int i = 0; i < policies.length; ++i) {
+                    VariablePropagationPolicy p = policies[i];
+                    IOperatorSchema s = sources[i];
+                    p.propagateVariables(target, s);
+                }
+            }
+        };
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
new file mode 100644
index 0000000..f3724a5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.scripting;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IScriptDescription {
+    public enum ScriptKind {
+        STRING_STREAMING
+    }
+
+    public ScriptKind getKind();
+
+    public List<Pair<LogicalVariable, Object>> getVarTypePairs();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
new file mode 100644
index 0000000..f030faf
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.scripting;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+
+public class StringStreamingScriptDescription implements IScriptDescription {
+
+    private final String command;
+    private final IPrinterFactory[] printerFactories;
+    private final char fieldDelimiter;
+    private final ITupleParserFactory parserFactory;
+    private final List<Pair<LogicalVariable, Object>> varTypePairs;
+
+    public StringStreamingScriptDescription(String command, IPrinterFactory[] printerFactories, char fieldDelimiter,
+            ITupleParserFactory parserFactory, List<Pair<LogicalVariable, Object>> varTypePairs) {
+        this.command = command;
+        this.printerFactories = printerFactories;
+        this.fieldDelimiter = fieldDelimiter;
+        this.parserFactory = parserFactory;
+        this.varTypePairs = varTypePairs;
+    }
+
+    @Override
+    public ScriptKind getKind() {
+        return ScriptKind.STRING_STREAMING;
+    }
+
+    public String getCommand() {
+        return command;
+    }
+
+    public IPrinterFactory[] getPrinterFactories() {
+        return printerFactories;
+    }
+
+    public char getFieldDelimiter() {
+        return fieldDelimiter;
+    }
+
+    public ITupleParserFactory getParserFactory() {
+        return parserFactory;
+    }
+
+    @Override
+    public List<Pair<LogicalVariable, Object>> getVarTypePairs() {
+        return varTypePairs;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
new file mode 100644
index 0000000..9833c17
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public abstract class AbstractTypeEnvironment implements IVariableTypeEnvironment {
+
+    protected final Map<LogicalVariable, Object> varTypeMap = new HashMap<LogicalVariable, Object>();
+    protected final IExpressionTypeComputer expressionTypeComputer;
+    protected final IMetadataProvider<?, ?> metadataProvider;
+
+    public AbstractTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
+            IMetadataProvider<?, ?> metadataProvider) {
+        this.expressionTypeComputer = expressionTypeComputer;
+        this.metadataProvider = metadataProvider;
+    }
+
+    @Override
+    public Object getType(ILogicalExpression expr) throws AlgebricksException {
+        return expressionTypeComputer.getType(expr, metadataProvider, this);
+    }
+
+    @Override
+    public void setVarType(LogicalVariable var, Object type) {
+        varTypeMap.put(var, type);
+    }
+
+    @Override
+    public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException {
+        Object t = varTypeMap.get(v1);
+        if (t == null) {
+            return false;
+        }
+        varTypeMap.put(v1, null);
+        varTypeMap.put(v2, t);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
new file mode 100644
index 0000000..0b56db7
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public interface ITypeEnvPointer {
+    public IVariableTypeEnvironment getTypeEnv();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypingContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypingContext.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypingContext.java
new file mode 100644
index 0000000..7945fa7
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/ITypingContext.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface ITypingContext {
+    public abstract IVariableTypeEnvironment getOutputTypeEnvironment(ILogicalOperator op);
+
+    public abstract void setOutputTypeEnvironment(ILogicalOperator op, IVariableTypeEnvironment env);
+
+    public abstract IExpressionTypeComputer getExpressionTypeComputer();
+
+    public abstract INullableTypeComputer getNullableTypeComputer();
+
+    public abstract IMetadataProvider<?, ?> getMetadataProvider();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
new file mode 100644
index 0000000..eb0eb71
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public class NonPropagatingTypeEnvironment extends AbstractTypeEnvironment {
+
+    public NonPropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
+            IMetadataProvider<?, ?> metadataProvider) {
+        super(expressionTypeComputer, metadataProvider);
+    }
+
+    @Override
+    public Object getVarType(LogicalVariable var) throws AlgebricksException {
+        return varTypeMap.get(var);
+    }
+
+    @Override
+    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
+            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+        return getVarType(var);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
new file mode 100644
index 0000000..6a1c8b5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public class OpRefTypeEnvPointer implements ITypeEnvPointer {
+
+    private final Mutable<ILogicalOperator> op;
+    private final ITypingContext ctx;
+
+    public OpRefTypeEnvPointer(Mutable<ILogicalOperator> op, ITypingContext ctx) {
+        this.op = op;
+        this.ctx = ctx;
+    }
+
+    @Override
+    public IVariableTypeEnvironment getTypeEnv() {
+        return ctx.getOutputTypeEnvironment(op.getValue());
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getName() + ":" + op;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
new file mode 100644
index 0000000..87ee385
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public class PropagateOperatorInputsTypeEnvironment extends AbstractTypeEnvironment {
+
+    private final List<LogicalVariable> nonNullVariables = new ArrayList<LogicalVariable>();
+    private final List<List<LogicalVariable>> correlatedNullableVariableLists = new ArrayList<List<LogicalVariable>>();
+    private final ILogicalOperator op;
+    private final ITypingContext ctx;
+
+    public PropagateOperatorInputsTypeEnvironment(ILogicalOperator op, ITypingContext ctx,
+            IExpressionTypeComputer expressionTypeComputer, IMetadataProvider<?, ?> metadataProvider) {
+        super(expressionTypeComputer, metadataProvider);
+        this.op = op;
+        this.ctx = ctx;
+    }
+
+    public List<LogicalVariable> getNonNullVariables() {
+        return nonNullVariables;
+    }
+
+    @Override
+    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
+            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+        nonNullVariableList.addAll(nonNullVariables);
+        return getVarTypeFullList(var, nonNullVariableList, correlatedNullableVariableLists);
+    }
+
+    private Object getVarTypeFullList(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
+            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+        Object t = varTypeMap.get(var);
+        if (t != null) {
+            return t;
+        }
+        for (Mutable<ILogicalOperator> r : op.getInputs()) {
+            ILogicalOperator c = r.getValue();
+            IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(c);
+            Object t2 = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists);
+            if (t2 != null) {
+                return t2;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public Object getVarType(LogicalVariable var) throws AlgebricksException {
+        return getVarTypeFullList(var, nonNullVariables, correlatedNullableVariableLists);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
new file mode 100644
index 0000000..d167cd7
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+
+public class PropagatingTypeEnvironment extends AbstractTypeEnvironment {
+
+    private final TypePropagationPolicy policy;
+
+    private final INullableTypeComputer nullableTypeComputer;
+
+    private final ITypeEnvPointer[] envPointers;
+
+    private final List<LogicalVariable> nonNullVariables = new ArrayList<LogicalVariable>();
+
+    private final List<List<LogicalVariable>> correlatedNullableVariableLists = new ArrayList<List<LogicalVariable>>();
+
+    public PropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
+            INullableTypeComputer nullableTypeComputer, IMetadataProvider<?, ?> metadataProvider,
+            TypePropagationPolicy policy, ITypeEnvPointer[] envPointers) {
+        super(expressionTypeComputer, metadataProvider);
+        this.nullableTypeComputer = nullableTypeComputer;
+        this.policy = policy;
+        this.envPointers = envPointers;
+    }
+
+    @Override
+    public Object getVarType(LogicalVariable var) throws AlgebricksException {
+        return getVarTypeFullList(var, nonNullVariables, correlatedNullableVariableLists);
+    }
+
+    public List<LogicalVariable> getNonNullVariables() {
+        return nonNullVariables;
+    }
+
+    public List<List<LogicalVariable>> getCorrelatedNullableVariableLists() {
+        return correlatedNullableVariableLists;
+    }
+
+    @Override
+    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
+            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+        for (LogicalVariable v : nonNullVariables) {
+            if (!nonNullVariableList.contains(v)) {
+                nonNullVariableList.add(v);
+            }
+        }
+        Object t = getVarTypeFullList(var, nonNullVariableList, correlatedNullableVariableLists);
+        for (List<LogicalVariable> list : this.correlatedNullableVariableLists) {
+            if (!correlatedNullableVariableLists.contains(list)) {
+                correlatedNullableVariableLists.add(list);
+            }
+        }
+        return t;
+    }
+
+    private Object getVarTypeFullList(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
+            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
+        Object t = varTypeMap.get(var);
+        if (t != null) {
+            return t;
+        }
+        return policy.getVarType(var, nullableTypeComputer, nonNullVariableList, correlatedNullableVariableLists,
+                envPointers);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
new file mode 100644
index 0000000..adb7a8f
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -0,0 +1,225 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.OperatorDeepCopyVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+
+public class OperatorManipulationUtil {
+
+    // Transforms all NestedTupleSource operators to EmptyTupleSource operators
+    public static void ntsToEts(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
+            context.computeAndSetTypeEnvironmentForOperator(ets);
+            opRef.setValue(ets);
+        } else {
+            for (Mutable<ILogicalOperator> i : opRef.getValue().getInputs()) {
+                ntsToEts(i, context);
+            }
+        }
+    }
+
+    public static ILogicalOperator eliminateSingleSubplanOverEts(SubplanOperator subplan) {
+        if (subplan.getNestedPlans().size() > 1) {
+            // not a single subplan
+            List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
+            subInpList.clear();
+            subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
+            return subplan;
+        }
+        ILogicalPlan plan = subplan.getNestedPlans().get(0);
+        if (plan.getRoots().size() > 1) {
+            // not a single subplan
+            List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
+            subInpList.clear();
+            subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
+            return subplan;
+        }
+        return plan.getRoots().get(0).getValue();
+    }
+
+    public static boolean setOperatorMode(AbstractLogicalOperator op) {
+        boolean change = false;
+        switch (op.getOperatorTag()) {
+            case DATASOURCESCAN: {
+                op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+                AbstractLogicalOperator currentOp = op;
+                while (currentOp.getInputs().size() == 1) {
+                    AbstractLogicalOperator child = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue();
+                    if (child.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+                        break;
+                    }
+                    child.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+                    currentOp = child;
+                }
+                change = true;
+                break;
+            }
+            case NESTEDTUPLESOURCE: {
+                NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
+                AbstractLogicalOperator prevOp = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
+                        .getInputs().get(0).getValue();
+                if (prevOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+                    nts.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+                    change = true;
+                }
+                break;
+            }
+            default: {
+                boolean forceUnpartitioned = false;
+                if (op.getOperatorTag() == LogicalOperatorTag.LIMIT) {
+                    LimitOperator opLim = (LimitOperator) op;
+                    if (opLim.isTopmostLimitOp()) {
+                        opLim.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
+                        change = true;
+                        forceUnpartitioned = true;
+                    }
+                }
+                if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+                    AggregateOperator aggOp = (AggregateOperator) op;
+                    if (aggOp.isGlobal()) {
+                        op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
+                        change = true;
+                        forceUnpartitioned = true;
+                    }
+                }
+
+                for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                    boolean exit = false;
+                    AbstractLogicalOperator inputOp = (AbstractLogicalOperator) i.getValue();
+                    switch (inputOp.getExecutionMode()) {
+                        case PARTITIONED: {
+                            if (forceUnpartitioned)
+                                break;
+                            op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+                            change = true;
+                            exit = true;
+                            break;
+                        }
+                        case LOCAL: {
+                            op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+                            change = true;
+                            break;
+                        }
+                    }
+                    if (exit) {
+                        break;
+                    }
+                }
+                break;
+            }
+        }
+        return change;
+    }
+
+    public static void substituteVarRec(AbstractLogicalOperator op, LogicalVariable v1, LogicalVariable v2,
+            boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
+        VariableUtilities.substituteVariables(op, v1, v2, goThroughNts, ctx);
+        for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
+            substituteVarRec((AbstractLogicalOperator) opRef2.getValue(), v1, v2, goThroughNts, ctx);
+        }
+        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE && goThroughNts) {
+            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
+            if (nts.getDataSourceReference() != null) {
+                AbstractLogicalOperator op2 = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
+                        .getInputs().get(0).getValue();
+                substituteVarRec(op2, v1, v2, goThroughNts, ctx);
+            }
+        }
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans aonp = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : aonp.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> ref : p.getRoots()) {
+                    AbstractLogicalOperator aop = (AbstractLogicalOperator) ref.getValue();
+                    substituteVarRec(aop, v1, v2, goThroughNts, ctx);
+                }
+            }
+        }
+    }
+
+    public static ILogicalPlan deepCopy(ILogicalPlan plan) throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> roots = plan.getRoots();
+        List<Mutable<ILogicalOperator>> newRoots = clonePipeline(roots);
+        return new ALogicalPlanImpl(newRoots);
+    }
+
+    public static ILogicalPlan deepCopy(ILogicalPlan plan, IOptimizationContext ctx) throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> roots = plan.getRoots();
+        List<Mutable<ILogicalOperator>> newRoots = clonePipeline(roots);
+        cloneTypeEnvironments(ctx, roots, newRoots);
+        return new ALogicalPlanImpl(newRoots);
+    }
+
+    private static List<Mutable<ILogicalOperator>> clonePipeline(List<Mutable<ILogicalOperator>> roots)
+            throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> newRoots = new ArrayList<Mutable<ILogicalOperator>>();
+        for (Mutable<ILogicalOperator> opRef : roots) {
+            newRoots.add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(opRef.getValue())));
+        }
+        return newRoots;
+    }
+
+    private static void cloneTypeEnvironments(IOptimizationContext ctx, List<Mutable<ILogicalOperator>> roots,
+            List<Mutable<ILogicalOperator>> newRoots) {
+        for (int i = 0; i < newRoots.size(); i++) {
+            Mutable<ILogicalOperator> opRef = newRoots.get(i);
+            Mutable<ILogicalOperator> oldOpRef = roots.get(i);
+            while (opRef.getValue().getInputs().size() > 0) {
+                ctx.setOutputTypeEnvironment(opRef.getValue(), ctx.getOutputTypeEnvironment(oldOpRef.getValue()));
+                opRef = opRef.getValue().getInputs().get(0);
+                oldOpRef = oldOpRef.getValue().getInputs().get(0);
+            }
+            ctx.setOutputTypeEnvironment(opRef.getValue(), ctx.getOutputTypeEnvironment(oldOpRef.getValue()));
+        }
+    }
+
+    public static ILogicalOperator bottomUpCopyOperators(ILogicalOperator op) throws AlgebricksException {
+        ILogicalOperator newOp = deepCopy(op);
+        newOp.getInputs().clear();
+        for (Mutable<ILogicalOperator> child : op.getInputs())
+            newOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(child.getValue())));
+        return newOp;
+    }
+
+    public static ILogicalOperator deepCopy(ILogicalOperator op) throws AlgebricksException {
+        OperatorDeepCopyVisitor visitor = new OperatorDeepCopyVisitor();
+        return op.accept(visitor, null);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
new file mode 100644
index 0000000..ec9b403
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
@@ -0,0 +1,300 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.util;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+
+public class OperatorPropertiesUtil {
+
+    public static <T> boolean disjoint(Collection<T> c1, Collection<T> c2) {
+        for (T m : c1) {
+            if (c2.contains(m)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    // Obs: doesn't return expected result for op. with nested plans.
+    private static void getFreeVariablesInOp(ILogicalOperator op, Set<LogicalVariable> freeVars)
+            throws AlgebricksException {
+        VariableUtilities.getUsedVariables(op, freeVars);
+        HashSet<LogicalVariable> produced = new HashSet<LogicalVariable>();
+        VariableUtilities.getProducedVariables(op, produced);
+        for (LogicalVariable v : produced) {
+            freeVars.remove(v);
+        }
+    }
+
+    /**
+     * Adds the free variables of the plan rooted at that operator to the
+     * collection provided.
+     * 
+     * @param op
+     * @param vars
+     *            - The collection to which the free variables will be added.
+     */
+    public static void getFreeVariablesInSelfOrDesc(AbstractLogicalOperator op, Set<LogicalVariable> freeVars)
+            throws AlgebricksException {
+        HashSet<LogicalVariable> produced = new HashSet<LogicalVariable>();
+        VariableUtilities.getProducedVariables(op, produced);
+        for (LogicalVariable v : produced) {
+            freeVars.remove(v);
+        }
+
+        HashSet<LogicalVariable> used = new HashSet<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op, used);
+        for (LogicalVariable v : used) {
+            if (!freeVars.contains(v)) {
+                freeVars.add(v);
+            }
+        }
+
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans s = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : s.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
+                }
+            }
+            s.getUsedVariablesExceptNestedPlans(freeVars);
+            HashSet<LogicalVariable> produced2 = new HashSet<LogicalVariable>();
+            s.getProducedVariablesExceptNestedPlans(produced2);
+            freeVars.removeAll(produced);
+        }
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) i.getValue(), freeVars);
+        }
+    }
+
+    /**
+     * Adds the free variables of the operator path from
+     * op to dest, where dest is a direct/indirect input operator of op in the query plan.
+     * 
+     * @param op
+     *            , the start operator.
+     * @param dest
+     *            , the destination operator (a direct/indirect input operator).
+     * @param freeVars
+     *            - The collection to which the free variables will be added.
+     */
+    public static void getFreeVariablesInPath(ILogicalOperator op, ILogicalOperator dest, Set<LogicalVariable> freeVars)
+            throws AlgebricksException {
+        Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
+        VariableUtilities.getLiveVariables(op, freeVars);
+        collectUsedAndProducedVariablesInPath(op, dest, freeVars, producedVars);
+        freeVars.removeAll(producedVars);
+    }
+
+    /**
+     * @param op
+     *            , the start operator.
+     * @param dest
+     *            , the destination operator (a direct/indirect input operator).
+     * @param usedVars
+     *            , the collection of used variables.
+     * @param producedVars
+     *            , the collection of produced variables.
+     * @return if the current operator is on the path from the original start operator to the destination operator.
+     * @throws AlgebricksException
+     */
+    private static boolean collectUsedAndProducedVariablesInPath(ILogicalOperator op, ILogicalOperator dest,
+            Set<LogicalVariable> usedVars, Set<LogicalVariable> producedVars) throws AlgebricksException {
+        if (op == dest) {
+            return true;
+        }
+        boolean onPath = false;
+        if (((AbstractLogicalOperator) op).hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : a.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    if (isDestInNestedPath((AbstractLogicalOperator) r.getValue(), dest)) {
+                        onPath = true;
+                    }
+                }
+            }
+        }
+        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+            if (collectUsedAndProducedVariablesInPath(childRef.getValue(), dest, usedVars, producedVars)) {
+                onPath = true;
+            }
+        }
+        if (onPath) {
+            VariableUtilities.getUsedVariables(op, usedVars);
+            VariableUtilities.getProducedVariables(op, producedVars);
+        }
+        return onPath;
+    }
+
+    /***
+     * Recursively checks if the dest operator is in the path of a nested plan
+     * 
+     * @param op
+     * @param dest
+     * @return
+     */
+    private static boolean isDestInNestedPath(AbstractLogicalOperator op, ILogicalOperator dest) {
+        if (op == dest) {
+            return true;
+        }
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            if (isDestInNestedPath((AbstractLogicalOperator) i.getValue(), dest)) {
+                return true;
+            }
+        }
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : a.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    if (isDestInNestedPath((AbstractLogicalOperator) r.getValue(), dest)) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
+    public static void getFreeVariablesInSubplans(AbstractOperatorWithNestedPlans op, Set<LogicalVariable> freeVars)
+            throws AlgebricksException {
+        for (ILogicalPlan p : op.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
+            }
+        }
+    }
+
+    public static boolean hasFreeVariablesInSelfOrDesc(AbstractLogicalOperator op) throws AlgebricksException {
+        HashSet<LogicalVariable> free = new HashSet<LogicalVariable>();
+        getFreeVariablesInSelfOrDesc(op, free);
+        return !free.isEmpty();
+    }
+
+    public static boolean hasFreeVariables(ILogicalOperator op) throws AlgebricksException {
+        HashSet<LogicalVariable> free = new HashSet<LogicalVariable>();
+        getFreeVariablesInOp(op, free);
+        return !free.isEmpty();
+    }
+
+    public static void computeSchemaAndPropertiesRecIfNull(AbstractLogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        if (op.getSchema() == null) {
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) i.getValue(), context);
+            }
+            if (op.hasNestedPlans()) {
+                AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+                for (ILogicalPlan p : a.getNestedPlans()) {
+                    for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                        computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) r.getValue(), context);
+                    }
+                }
+            }
+            op.recomputeSchema();
+            op.computeDeliveredPhysicalProperties(context);
+        }
+    }
+
+    public static void computeSchemaRecIfNull(AbstractLogicalOperator op) throws AlgebricksException {
+        if (op.getSchema() == null) {
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                computeSchemaRecIfNull((AbstractLogicalOperator) i.getValue());
+            }
+            if (op.hasNestedPlans()) {
+                AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+                for (ILogicalPlan p : a.getNestedPlans()) {
+                    for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                        computeSchemaRecIfNull((AbstractLogicalOperator) r.getValue());
+                    }
+                }
+            }
+            op.recomputeSchema();
+        }
+    }
+
+    public static boolean isNullTest(AbstractLogicalOperator op) {
+        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+        AbstractLogicalOperator doubleUnder = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        if (doubleUnder.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            return false;
+        }
+        ILogicalExpression eu = ((SelectOperator) op).getCondition().getValue();
+        if (eu.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) eu;
+        if (!f1.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
+            return false;
+        }
+        ILogicalExpression a1 = f1.getArguments().get(0).getValue();
+        if (!a1.getExpressionTag().equals(LogicalExpressionTag.FUNCTION_CALL)) {
+            return false;
+        }
+        AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
+        if (!f2.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
+            return false;
+        }
+        return true;
+    }
+
+    public static void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
+            typeOpRec(r, context);
+        }
+    }
+
+    public static void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            typeOpRec(i, context);
+        }
+        if (op.hasNestedPlans()) {
+            for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+                typePlan(p, context);
+            }
+        }
+        context.computeAndSetTypeEnvironmentForOperator(op);
+    }
+
+    public static boolean isAlwaysTrueCond(ILogicalExpression cond) {
+        if (cond.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+            return ((ConstantExpression) cond).getValue().isTrue();
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
new file mode 100644
index 0000000..c7424f0
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+
+public abstract class AbstractConstVarFunVisitor<R, T> implements ILogicalExpressionVisitor<R, T> {
+
+    public abstract R visitFunctionCallExpression(AbstractFunctionCallExpression expr, T arg)
+            throws AlgebricksException;
+
+    public R visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, T arg)
+            throws AlgebricksException {
+        return visitFunctionCallExpression(expr, arg);
+    }
+
+    public R visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, T arg) throws AlgebricksException {
+        return visitFunctionCallExpression(expr, arg);
+    }
+
+    public R visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, T arg) throws AlgebricksException {
+        return visitFunctionCallExpression(expr, arg);
+    }
+
+    public R visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, T arg)
+            throws AlgebricksException {
+        return visitFunctionCallExpression(expr, arg);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
new file mode 100644
index 0000000..06cec65
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
+public interface ILogicalExpressionReferenceTransform {
+    // returns true if any change is made to the expression
+    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
new file mode 100644
index 0000000..3448db8
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+
+public interface ILogicalExpressionVisitor<R, T> {
+    public R visitConstantExpression(ConstantExpression expr, T arg) throws AlgebricksException;
+
+    public R visitVariableReferenceExpression(VariableReferenceExpression expr, T arg) throws AlgebricksException;
+
+    public R visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, T arg)
+            throws AlgebricksException;
+
+    public R visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, T arg) throws AlgebricksException;
+
+    public R visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, T arg) throws AlgebricksException;
+
+    public R visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, T arg)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
new file mode 100644
index 0000000..359e243
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+
+public interface ILogicalOperatorVisitor<R, T> {
+
+    public R visitAggregateOperator(AggregateOperator op, T arg) throws AlgebricksException;
+
+    public R visitRunningAggregateOperator(RunningAggregateOperator op, T arg) throws AlgebricksException;
+
+    public R visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, T arg) throws AlgebricksException;
+
+    public R visitGroupByOperator(GroupByOperator op, T arg) throws AlgebricksException;
+
+    public R visitLimitOperator(LimitOperator op, T arg) throws AlgebricksException;
+
+    public R visitInnerJoinOperator(InnerJoinOperator op, T arg) throws AlgebricksException;
+
+    public R visitLeftOuterJoinOperator(LeftOuterJoinOperator op, T arg) throws AlgebricksException;
+
+    public R visitNestedTupleSourceOperator(NestedTupleSourceOperator op, T arg) throws AlgebricksException;
+
+    public R visitOrderOperator(OrderOperator op, T arg) throws AlgebricksException;
+
+    public R visitAssignOperator(AssignOperator op, T arg) throws AlgebricksException;
+
+    public R visitSelectOperator(SelectOperator op, T arg) throws AlgebricksException;
+
+    public R visitExtensionOperator(ExtensionOperator op, T arg) throws AlgebricksException;
+
+    public R visitProjectOperator(ProjectOperator op, T arg) throws AlgebricksException;
+
+    public R visitPartitioningSplitOperator(PartitioningSplitOperator op, T arg) throws AlgebricksException;
+
+    public R visitReplicateOperator(ReplicateOperator op, T arg) throws AlgebricksException;
+
+    public R visitMaterializeOperator(MaterializeOperator op, T arg) throws AlgebricksException;
+
+    public R visitScriptOperator(ScriptOperator op, T arg) throws AlgebricksException;
+
+    public R visitSubplanOperator(SubplanOperator op, T arg) throws AlgebricksException;
+
+    public R visitSinkOperator(SinkOperator op, T arg) throws AlgebricksException;
+
+    public R visitUnionOperator(UnionAllOperator op, T arg) throws AlgebricksException;
+
+    public R visitUnnestOperator(UnnestOperator op, T arg) throws AlgebricksException;
+
+    public R visitUnnestMapOperator(UnnestMapOperator op, T arg) throws AlgebricksException;
+
+    public R visitDataScanOperator(DataSourceScanOperator op, T arg) throws AlgebricksException;
+
+    public R visitDistinctOperator(DistinctOperator op, T arg) throws AlgebricksException;
+
+    public R visitExchangeOperator(ExchangeOperator op, T arg) throws AlgebricksException;
+
+    public R visitWriteOperator(WriteOperator op, T arg) throws AlgebricksException;
+
+    public R visitDistributeResultOperator(DistributeResultOperator op, T arg) throws AlgebricksException;
+
+    public R visitWriteResultOperator(WriteResultOperator op, T arg) throws AlgebricksException;
+
+    public R visitInsertDeleteOperator(InsertDeleteOperator op, T tag) throws AlgebricksException;
+
+    public R visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, T tag) throws AlgebricksException;
+
+    public R visitExternalDataLookupOperator(ExternalDataLookupOperator op, T arg) throws AlgebricksException;
+
+    public R visitTokenizeOperator(TokenizeOperator op, T arg) throws AlgebricksException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
new file mode 100644
index 0000000..abdd6d1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.config;
+
+import java.util.logging.Logger;
+
+public class AlgebricksConfig {
+    public static final boolean DEBUG = true;
+    public static final String ALGEBRICKS_LOGGER_NAME = "edu.uci.ics.hyracks.algebricks";
+    public static final Logger ALGEBRICKS_LOGGER = Logger.getLogger(ALGEBRICKS_LOGGER_NAME);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/SysoutFormatter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/SysoutFormatter.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/SysoutFormatter.java
new file mode 100644
index 0000000..d449f92
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/SysoutFormatter.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.config;
+
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.LogRecord;
+
+public class SysoutFormatter extends Formatter {
+    public String format(LogRecord rec) {
+        StringBuffer buf = new StringBuffer(5000);
+        buf.append(formatMessage(rec));
+        return buf.toString();
+    }
+
+    public String getHead(Handler h) {
+        return "";
+    }
+
+    public String getTail(Handler h) {
+        return "";
+    }
+}
\ No newline at end of file

[04/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
new file mode 100644
index 0000000..bc7255b
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -0,0 +1,424 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AggregatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.BulkloadPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.DataSourceScanPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.DistributeResultPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.EmptyTupleSourcePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.IndexBulkloadPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.IndexInsertDeletePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InsertDeletePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.MicroPreclusteredGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NestedTupleSourcePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreSortedDistinctByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RunningAggregatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SinkPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SinkWritePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamSelectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StringStreamingScriptPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SubplanPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.TokenizePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnionAllPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnnestPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.WriteResultPOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.JoinUtils;
+
+public class SetAlgebricksPhysicalOperatorsRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        // if (context.checkIfInDontApplySet(this, op)) {
+        // return false;
+        // }
+        if (op.getPhysicalOperator() != null) {
+            return false;
+        }
+
+        computeDefaultPhysicalOp(op, true, context);
+        // context.addToDontApplySet(this, op);
+        return true;
+    }
+
+    private static void setPhysicalOperators(ILogicalPlan plan, boolean topLevelOp, IOptimizationContext context)
+            throws AlgebricksException {
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            computeDefaultPhysicalOp((AbstractLogicalOperator) root.getValue(), topLevelOp, context);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, boolean topLevelOp,
+            IOptimizationContext context) throws AlgebricksException {
+        PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+        if (op.getPhysicalOperator() == null) {
+            switch (op.getOperatorTag()) {
+                case AGGREGATE: {
+                    op.setPhysicalOperator(new AggregatePOperator());
+                    break;
+                }
+                case ASSIGN: {
+                    op.setPhysicalOperator(new AssignPOperator());
+                    break;
+                }
+                case DISTINCT: {
+                    DistinctOperator distinct = (DistinctOperator) op;
+                    distinct.setPhysicalOperator(new PreSortedDistinctByPOperator(distinct.getDistinctByVarList()));
+                    break;
+                }
+                case EMPTYTUPLESOURCE: {
+                    op.setPhysicalOperator(new EmptyTupleSourcePOperator());
+                    break;
+                }
+                case EXCHANGE: {
+                    if (op.getPhysicalOperator() == null) {
+                        throw new AlgebricksException("Implementation for EXCHANGE operator was not set.");
+                    }
+                    // implem. choice for exchange should be set by a parent op.
+                    break;
+                }
+                case GROUP: {
+                    GroupByOperator gby = (GroupByOperator) op;
+
+                    if (gby.getNestedPlans().size() == 1) {
+                        ILogicalPlan p0 = gby.getNestedPlans().get(0);
+                        if (p0.getRoots().size() == 1) {
+                            if (gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE
+                                    || gby.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE) {
+                                if (!topLevelOp) {
+                                    throw new NotImplementedException(
+                                            "External hash group-by for nested grouping is not implemented.");
+                                }
+
+                                boolean hasIntermediateAgg = generateMergeAggregationExpressions(gby, context);
+                                if (hasIntermediateAgg) {
+                                    ExternalGroupByPOperator externalGby = new ExternalGroupByPOperator(
+                                            gby.getGroupByList(),
+                                            physicalOptimizationConfig.getMaxFramesExternalGroupBy(),
+                                            physicalOptimizationConfig.getExternalGroupByTableSize());
+                                    op.setPhysicalOperator(externalGby);
+                                    break;
+                                }
+                            }
+                        }
+                    }
+
+                    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
+                    List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
+                    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+                        ILogicalExpression expr = p.second.getValue();
+                        if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+                            columnList.add(varRef.getVariableReference());
+                        }
+                    }
+                    if (topLevelOp) {
+                        op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList));
+                    } else {
+                        op.setPhysicalOperator(new MicroPreclusteredGroupByPOperator(columnList));
+                    }
+                    break;
+                }
+                case INNERJOIN: {
+                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, context);
+                    break;
+                }
+                case LEFTOUTERJOIN: {
+                    JoinUtils.setJoinAlgorithmAndExchangeAlgo((LeftOuterJoinOperator) op, context);
+                    break;
+                }
+                case LIMIT: {
+                    op.setPhysicalOperator(new StreamLimitPOperator());
+                    break;
+                }
+                case NESTEDTUPLESOURCE: {
+                    op.setPhysicalOperator(new NestedTupleSourcePOperator());
+                    break;
+                }
+                case ORDER: {
+                    OrderOperator oo = (OrderOperator) op;
+                    for (Pair<IOrder, Mutable<ILogicalExpression>> p : oo.getOrderExpressions()) {
+                        ILogicalExpression e = p.second.getValue();
+                        if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                            throw new AlgebricksException("Order expression " + e + " has not been normalized.");
+                        }
+                    }
+                    if (topLevelOp) {
+                        op.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig
+                                .getMaxFramesExternalSort()));
+                    } else {
+                        op.setPhysicalOperator(new InMemoryStableSortPOperator());
+                    }
+                    break;
+                }
+                case PROJECT: {
+                    op.setPhysicalOperator(new StreamProjectPOperator());
+                    break;
+                }
+                case RUNNINGAGGREGATE: {
+                    op.setPhysicalOperator(new RunningAggregatePOperator());
+                    break;
+                }
+                case REPLICATE: {
+                    op.setPhysicalOperator(new ReplicatePOperator());
+                    break;
+                }
+                case SCRIPT: {
+                    op.setPhysicalOperator(new StringStreamingScriptPOperator());
+                    break;
+                }
+                case SELECT: {
+                    op.setPhysicalOperator(new StreamSelectPOperator());
+                    break;
+                }
+                case SUBPLAN: {
+                    op.setPhysicalOperator(new SubplanPOperator());
+                    break;
+                }
+                case UNIONALL: {
+                    op.setPhysicalOperator(new UnionAllPOperator());
+                    break;
+                }
+
+                case UNNEST: {
+                    op.setPhysicalOperator(new UnnestPOperator());
+                    break;
+                }
+                case DATASOURCESCAN: {
+                    DataSourceScanOperator scan = (DataSourceScanOperator) op;
+                    IDataSource dataSource = scan.getDataSource();
+                    DataSourceScanPOperator dss = new DataSourceScanPOperator(dataSource);
+                    IMetadataProvider mp = context.getMetadataProvider();
+                    if (mp.scannerOperatorIsLeaf(dataSource)) {
+                        dss.disableJobGenBelowMe();
+                    }
+                    op.setPhysicalOperator(dss);
+                    break;
+                }
+                case WRITE: {
+                    op.setPhysicalOperator(new SinkWritePOperator());
+                    break;
+                }
+                case DISTRIBUTE_RESULT: {
+                    op.setPhysicalOperator(new DistributeResultPOperator());
+                    break;
+                }
+                case WRITE_RESULT: {
+                    WriteResultOperator opLoad = (WriteResultOperator) op;
+                    LogicalVariable payload;
+                    List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
+                    List<LogicalVariable> additionalFilteringKeys = null;
+                    payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getKeyExpressions(), keys);
+                    if (opLoad.getAdditionalFilteringExpressions() != null) {
+                        additionalFilteringKeys = new ArrayList<LogicalVariable>();
+                        getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys);
+                    }
+                    op.setPhysicalOperator(new WriteResultPOperator(opLoad.getDataSource(), payload, keys,
+                            additionalFilteringKeys));
+                    break;
+                }
+                case INSERT_DELETE: {
+                    InsertDeleteOperator opLoad = (InsertDeleteOperator) op;
+                    LogicalVariable payload;
+                    List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
+                    List<LogicalVariable> additionalFilteringKeys = null;
+                    payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getPrimaryKeyExpressions(), keys);
+                    if (opLoad.getAdditionalFilteringExpressions() != null) {
+                        additionalFilteringKeys = new ArrayList<LogicalVariable>();
+                        getKeys(opLoad.getAdditionalFilteringExpressions(), additionalFilteringKeys);
+                    }
+                    if (opLoad.isBulkload()) {
+                        op.setPhysicalOperator(new BulkloadPOperator(payload, keys, additionalFilteringKeys, opLoad
+                                .getDataSource()));
+                    } else {
+                        op.setPhysicalOperator(new InsertDeletePOperator(payload, keys, additionalFilteringKeys, opLoad
+                                .getDataSource()));
+                    }
+                    break;
+                }
+                case INDEX_INSERT_DELETE: {
+                    IndexInsertDeleteOperator opInsDel = (IndexInsertDeleteOperator) op;
+                    List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
+                    List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
+                    List<LogicalVariable> additionalFilteringKeys = null;
+                    getKeys(opInsDel.getPrimaryKeyExpressions(), primaryKeys);
+                    getKeys(opInsDel.getSecondaryKeyExpressions(), secondaryKeys);
+                    if (opInsDel.getAdditionalFilteringExpressions() != null) {
+                        additionalFilteringKeys = new ArrayList<LogicalVariable>();
+                        getKeys(opInsDel.getAdditionalFilteringExpressions(), additionalFilteringKeys);
+                    }
+                    if (opInsDel.isBulkload()) {
+                        op.setPhysicalOperator(new IndexBulkloadPOperator(primaryKeys, secondaryKeys,
+                                additionalFilteringKeys, opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex()));
+                    } else {
+                        op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys,
+                                additionalFilteringKeys, opInsDel.getFilterExpression(), opInsDel.getDataSourceIndex()));
+                    }
+
+                    break;
+
+                }
+                case TOKENIZE: {
+                    TokenizeOperator opTokenize = (TokenizeOperator) op;
+                    List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
+                    List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
+                    getKeys(opTokenize.getPrimaryKeyExpressions(), primaryKeys);
+                    getKeys(opTokenize.getSecondaryKeyExpressions(), secondaryKeys);
+                    // Tokenize Operator only operates with a bulk load on a data set with an index
+                    if (opTokenize.isBulkload()) {
+                        op.setPhysicalOperator(new TokenizePOperator(primaryKeys, secondaryKeys, opTokenize
+                                .getDataSourceIndex()));
+                    }
+                    break;
+                }
+                case SINK: {
+                    op.setPhysicalOperator(new SinkPOperator());
+                    break;
+                }
+            }
+        }
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : nested.getNestedPlans()) {
+                setPhysicalOperators(p, false, context);
+            }
+        }
+        for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
+            computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), topLevelOp, context);
+        }
+    }
+
+    private static void getKeys(List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
+        for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
+            ILogicalExpression e = kExpr.getValue();
+            if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new NotImplementedException();
+            }
+            keys.add(((VariableReferenceExpression) e).getVariableReference());
+        }
+    }
+
+    private static LogicalVariable getKeysAndLoad(Mutable<ILogicalExpression> payloadExpr,
+            List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
+        LogicalVariable payload;
+        if (payloadExpr.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            throw new NotImplementedException();
+        }
+        payload = ((VariableReferenceExpression) payloadExpr.getValue()).getVariableReference();
+
+        for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
+            ILogicalExpression e = kExpr.getValue();
+            if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new NotImplementedException();
+            }
+            keys.add(((VariableReferenceExpression) e).getVariableReference());
+        }
+        return payload;
+    }
+
+    private static boolean generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context)
+            throws AlgebricksException {
+        if (gby.getNestedPlans().size() != 1) {
+            //External/Sort group-by currently works only for one nested plan with one root containing
+            //an aggregate and a nested-tuple-source.
+            throw new AlgebricksException(
+                    "External group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        ILogicalPlan p0 = gby.getNestedPlans().get(0);
+        if (p0.getRoots().size() != 1) {
+            //External/Sort group-by currently works only for one nested plan with one root containing
+            //an aggregate and a nested-tuple-source.
+            throw new AlgebricksException(
+                    "External group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
+                .getMergeAggregationExpressionFactory();
+        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+        AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
+        if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+        List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
+        List<LogicalVariable> originalAggVars = aggOp.getVariables();
+        int n = aggOp.getExpressions().size();
+        List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
+        for (int i = 0; i < n; i++) {
+            ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(
+                    originalAggVars.get(i), aggFuncRefs.get(i).getValue(), context);
+            if (mergeExpr == null) {
+                return false;
+            }
+            mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
+        }
+        aggOp.setMergeExpressions(mergeExpressionRefs);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
new file mode 100644
index 0000000..d938f73
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule sets the executionMode property of an operator, w/o introducing
+ * EXCHANGE operators in the plan. Previously, i.e. before having physical
+ * optimizations in place, we were using the IntroduceExchangeRule, which was
+ * doing both, to both set excutionMode and introduce data exchange ops.
+ * 
+ * @author Nicola
+ */
+public class SetExecutionModeRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        boolean changed = OperatorManipulationUtil.setOperatorMode(op);
+        if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED
+                || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) {
+            return changed;
+        }
+        switch (op.getOperatorTag()) {
+        // case DISTINCT:
+        // case AGGREGATE:
+        // case GROUP:
+        // case ORDER:
+        // case INNERJOIN:
+        // case LEFTOUTERJOIN: {
+        // op.setExecutionMode(ExecutionMode.GLOBAL);
+        // return true;
+        // }
+
+            case PARTITIONINGSPLIT: {
+                throw new NotImplementedException();
+            }
+            default: {
+                return changed;
+            }
+        }
+
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
new file mode 100644
index 0000000..f6c6ad8
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class SimpleUnnestToProductRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN
+                && op.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+        if (!(op2 instanceof AbstractScanOperator) && !descOrSelfIsSourceScan(op2)) {
+            return false;
+        }
+        // Make sure that op does not use any variables produced by op2.
+        if (!opsAreIndependent(op, op2)) {
+            return false;
+        }
+
+        /**
+         * finding the boundary between left branch and right branch
+         * operator pipeline on-top-of boundaryOpRef (exclusive) is the inner branch
+         * operator pipeline under boundaryOpRef (inclusive) is the outer branch
+         */
+        Mutable<ILogicalOperator> currentOpRef = opRef;
+        Mutable<ILogicalOperator> boundaryOpRef = currentOpRef.getValue().getInputs().get(0);
+        while (currentOpRef.getValue().getInputs().size() == 1) {
+            currentOpRef = currentOpRef.getValue().getInputs().get(0);
+        }
+        Mutable<ILogicalOperator> tupleSourceOpRef = currentOpRef;
+        currentOpRef = opRef;
+        if (tupleSourceOpRef.getValue().getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) tupleSourceOpRef.getValue();
+            // If the subplan input is a trivial plan, do not do the rewriting.
+            if (nts.getSourceOperator().getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE) {
+                while (currentOpRef.getValue().getInputs().size() == 1
+                        && currentOpRef.getValue() instanceof AbstractScanOperator
+                        && descOrSelfIsSourceScan((AbstractLogicalOperator) currentOpRef.getValue())) {
+                    if (opsAreIndependent(currentOpRef.getValue(), tupleSourceOpRef.getValue())) {
+                        /** move down the boundary if the operator is independent of the tuple source */
+                        boundaryOpRef = currentOpRef.getValue().getInputs().get(0);
+                    } else {
+                        break;
+                    }
+                    currentOpRef = currentOpRef.getValue().getInputs().get(0);
+                }
+            }
+        }
+
+        /** join the two independent branches */
+        InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE),
+                new MutableObject<ILogicalOperator>(boundaryOpRef.getValue()), new MutableObject<ILogicalOperator>(
+                        opRef.getValue()));
+        opRef.setValue(join);
+        ILogicalOperator ets = new EmptyTupleSourceOperator();
+        boundaryOpRef.setValue(ets);
+        context.computeAndSetTypeEnvironmentForOperator(boundaryOpRef.getValue());
+        context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
+        context.computeAndSetTypeEnvironmentForOperator(join);
+        return true;
+    }
+
+    private boolean descOrSelfIsSourceScan(AbstractLogicalOperator op2) {
+        // Disregard data source scans in a subplan.
+        if (op2.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN
+                && op2.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return true;
+        }
+        for (Mutable<ILogicalOperator> cRef : op2.getInputs()) {
+            AbstractLogicalOperator alo = (AbstractLogicalOperator) cRef.getValue();
+            if (descOrSelfIsSourceScan(alo)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean opsAreIndependent(ILogicalOperator unnestOp, ILogicalOperator outer) throws AlgebricksException {
+        if (unnestOp.equals(outer)) {
+            return false;
+        }
+        List<LogicalVariable> opUsedVars = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(unnestOp, opUsedVars);
+        Set<LogicalVariable> op2LiveVars = new HashSet<LogicalVariable>();
+        VariableUtilities.getLiveVariables(outer, op2LiveVars);
+        for (LogicalVariable usedVar : opUsedVars) {
+            if (op2LiveVars.contains(usedVar)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
new file mode 100644
index 0000000..f249122
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Looks for a nested group-by plan ending in
+ * subplan {
+ * ...
+ * }
+ * select (function-call: algebricks:not, Args:[function-call:
+ * algebricks:is-null, Args:[...]])
+ * nested tuple source -- |UNPARTITIONED|
+ */
+
+public class SubplanOutOfGroupRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+        if (op0.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        GroupByOperator gby = (GroupByOperator) op0;
+
+        Iterator<ILogicalPlan> plansIter = gby.getNestedPlans().iterator();
+        ILogicalPlan p = null;
+        while (plansIter.hasNext()) {
+            p = plansIter.next();
+        }
+        if (p == null) {
+            return false;
+        }
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> op1Ref = p.getRoots().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getValue();
+        boolean found = false;
+        while (op1.getInputs().size() == 1) {
+            if (op1.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+                SubplanOperator subplan = (SubplanOperator) op1;
+                AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getValue();
+                if (OperatorPropertiesUtil.isNullTest(op2)) {
+                    if (subplan.getNestedPlans().size() == 1) {
+                        ILogicalPlan p1 = subplan.getNestedPlans().get(0);
+                        if (p1.getRoots().size() == 1) {
+                            AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getValue();
+                            if (r1.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+                                    || r1.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+                                // now, check that it propagates all variables,
+                                // so it can be pushed
+                                List<LogicalVariable> op2Vars = new ArrayList<LogicalVariable>();
+                                VariableUtilities.getLiveVariables(op2, op2Vars);
+                                List<LogicalVariable> op1Vars = new ArrayList<LogicalVariable>();
+                                VariableUtilities.getLiveVariables(subplan, op1Vars);
+                                if (op1Vars.containsAll(op2Vars)) {
+                                    found = true;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            op1Ref = op1.getInputs().get(0);
+            op1 = (AbstractLogicalOperator) op1Ref.getValue();
+        }
+        if (!found) {
+            return false;
+        }
+
+        ILogicalOperator subplan = op1;
+        ILogicalOperator op2 = op1.getInputs().get(0).getValue();
+        op1Ref.setValue(op2);
+        Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0);
+        ILogicalOperator opUnder = opUnderRef.getValue();
+        subplan.getInputs().clear();
+        subplan.getInputs().add(new MutableObject<ILogicalOperator>(opUnder));
+        opUnderRef.setValue(subplan);
+
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
new file mode 100644
index 0000000..0881b50
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -0,0 +1,210 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalPropertiesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractJoinPOperator.JoinPartitioningType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HybridHashJoinPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryHashJoinPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NLJoinPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class JoinUtils {
+
+    public static void setJoinAlgorithmAndExchangeAlgo(AbstractBinaryJoinOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        List<LogicalVariable> sideLeft = new LinkedList<LogicalVariable>();
+        List<LogicalVariable> sideRight = new LinkedList<LogicalVariable>();
+        List<LogicalVariable> varsLeft = op.getInputs().get(0).getValue().getSchema();
+        List<LogicalVariable> varsRight = op.getInputs().get(1).getValue().getSchema();
+        if (isHashJoinCondition(op.getCondition().getValue(), varsLeft, varsRight, sideLeft, sideRight)) {
+            BroadcastSide side = getBroadcastJoinSide(op.getCondition().getValue(), varsLeft, varsRight);
+            if (side == null) {
+                setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+            } else {
+                switch (side) {
+                    case RIGHT:
+                        setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideLeft, sideRight, context);
+                        break;
+                    case LEFT:
+                        Mutable<ILogicalOperator> opRef0 = op.getInputs().get(0);
+                        Mutable<ILogicalOperator> opRef1 = op.getInputs().get(1);
+                        ILogicalOperator tmp = opRef0.getValue();
+                        opRef0.setValue(opRef1.getValue());
+                        opRef1.setValue(tmp);
+                        setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideRight, sideLeft, context);
+                        break;
+                    default:
+                        setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+                }
+            }
+        } else {
+            setNLJoinOp(op, context);
+        }
+    }
+
+    private static void setNLJoinOp(AbstractBinaryJoinOperator op, IOptimizationContext context) {
+        op.setPhysicalOperator(new NLJoinPOperator(op.getJoinKind(), JoinPartitioningType.BROADCAST, context
+                .getPhysicalOptimizationConfig().getMaxRecordsPerFrame()));
+    }
+
+    private static void setHashJoinOp(AbstractBinaryJoinOperator op, JoinPartitioningType partitioningType,
+            List<LogicalVariable> sideLeft, List<LogicalVariable> sideRight, IOptimizationContext context)
+            throws AlgebricksException {
+        op.setPhysicalOperator(new HybridHashJoinPOperator(op.getJoinKind(), partitioningType, sideLeft, sideRight,
+                context.getPhysicalOptimizationConfig().getMaxFramesHybridHash(), context
+                        .getPhysicalOptimizationConfig().getMaxFramesLeftInputHybridHash(), context
+                        .getPhysicalOptimizationConfig().getMaxRecordsPerFrame(), context
+                        .getPhysicalOptimizationConfig().getFudgeFactor()));
+        if (partitioningType == JoinPartitioningType.BROADCAST) {
+            hybridToInMemHashJoin(op, context);
+        }
+        // op.setPhysicalOperator(new
+        // InMemoryHashJoinPOperator(op.getJoinKind(), partitioningType,
+        // sideLeft, sideRight,
+        // 1024 * 512));
+    }
+
+    private static void hybridToInMemHashJoin(AbstractBinaryJoinOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator opBuild = op.getInputs().get(1).getValue();
+        LogicalPropertiesVisitor.computeLogicalPropertiesDFS(opBuild, context);
+        ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild);
+        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch -- Logical properties for " + opBuild
+                + ": " + v + "\n");
+        if (v != null) {
+            int size2 = v.getMaxOutputFrames();
+            HybridHashJoinPOperator hhj = (HybridHashJoinPOperator) op.getPhysicalOperator();
+            if (size2 > 0 && size2 * hhj.getFudgeFactor() <= hhj.getMemSizeInFrames()) {
+                AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch " + opBuild
+                        + " fits in memory\n");
+                // maintains the local properties on the probe side
+                op.setPhysicalOperator(new InMemoryHashJoinPOperator(hhj.getKind(), hhj.getPartitioningType(), hhj
+                        .getKeysLeftBranch(), hhj.getKeysRightBranch(), v.getNumberOfTuples() * 2));
+            }
+        }
+
+    }
+
+    private static boolean isHashJoinCondition(ILogicalExpression e, Collection<LogicalVariable> inLeftAll,
+            Collection<LogicalVariable> inRightAll, Collection<LogicalVariable> outLeftFields,
+            Collection<LogicalVariable> outRightFields) {
+        switch (e.getExpressionTag()) {
+            case FUNCTION_CALL: {
+                AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
+                FunctionIdentifier fi = fexp.getFunctionIdentifier();
+                if (fi.equals(AlgebricksBuiltinFunctions.AND)) {
+                    for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
+                        if (!isHashJoinCondition(a.getValue(), inLeftAll, inRightAll, outLeftFields,
+                                outRightFields)) {
+                            return false;
+                        }
+                    }
+                    return true;
+                } else {
+                    ComparisonKind ck = AlgebricksBuiltinFunctions.getComparisonType(fi);
+                    if (ck != ComparisonKind.EQ) {
+                        return false;
+                    }
+                    ILogicalExpression opLeft = fexp.getArguments().get(0).getValue();
+                    ILogicalExpression opRight = fexp.getArguments().get(1).getValue();
+                    if (opLeft.getExpressionTag() != LogicalExpressionTag.VARIABLE
+                            || opRight.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                        return false;
+                    }
+                    LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
+                    if (inLeftAll.contains(var1) && !outLeftFields.contains(var1)) {
+                        outLeftFields.add(var1);
+                    } else if (inRightAll.contains(var1) && !outRightFields.contains(var1)) {
+                        outRightFields.add(var1);
+                    } else {
+                        return false;
+                    }
+                    LogicalVariable var2 = ((VariableReferenceExpression) opRight).getVariableReference();
+                    if (inLeftAll.contains(var2) && !outLeftFields.contains(var2)) {
+                        outLeftFields.add(var2);
+                    } else if (inRightAll.contains(var2) && !outRightFields.contains(var2)) {
+                        outRightFields.add(var2);
+                    } else {
+                        return false;
+                    }
+                    return true;
+                }
+            }
+            default: {
+                return false;
+            }
+        }
+    }
+
+    private static BroadcastSide getBroadcastJoinSide(ILogicalExpression e, List<LogicalVariable> varsLeft,
+            List<LogicalVariable> varsRight) {
+        if (e.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return null;
+        }
+        AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
+        IExpressionAnnotation ann = fexp.getAnnotations().get(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY);
+        if (ann == null) {
+            return null;
+        }
+        BroadcastSide side = (BroadcastSide) ann.getObject();
+        if (side == null) {
+            return null;
+        }
+        int i;
+        switch (side) {
+            case LEFT:
+                i = 0;
+                break;
+            case RIGHT:
+                i = 1;
+                break;
+            default:
+                return null;
+        }
+        ArrayList<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+        fexp.getArguments().get(i).getValue().getUsedVariables(vars);
+        if (varsLeft.containsAll(vars)) {
+            return BroadcastSide.LEFT;
+        } else if (varsRight.containsAll(vars)) {
+            return BroadcastSide.RIGHT;
+        } else {
+            return null;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
new file mode 100644
index 0000000..ecacc05
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class PhysicalOptimizationsUtil {
+
+    public static void computeFDsAndEquivalenceClasses(AbstractLogicalOperator op, IOptimizationContext ctx)
+            throws AlgebricksException {
+        FDsAndEquivClassesVisitor visitor = new FDsAndEquivClassesVisitor();
+        Set<ILogicalOperator> visitSet = new HashSet<ILogicalOperator>();
+        computeFDsAndEqClassesWithVisitorRec(op, ctx, visitor, visitSet);
+    }
+
+    private static void computeFDsAndEqClassesWithVisitorRec(AbstractLogicalOperator op, IOptimizationContext ctx,
+            FDsAndEquivClassesVisitor visitor, Set<ILogicalOperator> visitSet) throws AlgebricksException {
+        visitSet.add(op);
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) i.getValue(), ctx, visitor, visitSet);
+        }
+        if (op.hasNestedPlans()) {
+            for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    AbstractLogicalOperator rootOp = (AbstractLogicalOperator) r.getValue();
+                    computeFDsAndEqClassesWithVisitorRec(rootOp, ctx, visitor, visitSet);
+                }
+            }
+        }
+        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
+            ILogicalOperator source = nts.getDataSourceReference().getValue().getInputs().get(0).getValue();
+            if (!visitSet.contains(source)) {
+                computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) source, ctx, visitor, visitSet);
+            }
+        }
+        op.accept(visitor, ctx);
+        if (AlgebricksConfig.DEBUG) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine("--> op. type = " + op.getOperatorTag() + "\n"
+                    + "    equiv. classes = " + ctx.getEquivalenceClassMap(op) + "\n" + "    FDs = "
+                    + ctx.getFDList(op) + "\n");
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java
deleted file mode 100644
index e6baffa..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountAggregateFunctionFactory.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.aggregators;
-
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class TupleCountAggregateFunctionFactory implements IAggregateEvaluatorFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    @Override
-    public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
-        final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-        return new IAggregateEvaluator() {
-
-            int cnt;
-
-            @Override
-            public void step(IFrameTupleReference tuple) throws AlgebricksException {
-                ++cnt;
-            }
-
-            @Override
-            public void init() throws AlgebricksException {
-                cnt = 0;
-            }
-
-            @Override
-            public void finish(IPointable result) throws AlgebricksException {
-                try {
-                    abvs.reset();
-                    abvs.getDataOutput().writeInt(cnt);
-                    result.set(abvs);
-                } catch (IOException e) {
-                    throw new AlgebricksException(e);
-                }
-            }
-
-            @Override
-            public void finishPartial(IPointable result) throws AlgebricksException {
-                finish(result);
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
deleted file mode 100644
index 37f2bca..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.aggregators;
-
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class TupleCountRunningAggregateFunctionFactory implements IRunningAggregateEvaluatorFactory {
-
-    private static final long serialVersionUID = 1L;
-
-    @Override
-    public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException {
-        final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-        return new IRunningAggregateEvaluator() {
-
-            int cnt;
-
-            @Override
-            public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
-                ++cnt;
-                try {
-                    abvs.reset();
-                    abvs.getDataOutput().writeInt(cnt);
-                    result.set(abvs);
-                } catch (IOException e) {
-                    throw new AlgebricksException(e);
-                }
-            }
-
-            @Override
-            public void init() throws AlgebricksException {
-                cnt = 0;
-            }
-        };
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java
deleted file mode 100644
index 1f24aa1..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/AlgebricksPipeline.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-
-public class AlgebricksPipeline implements Serializable {
-
-    private static final long serialVersionUID = 1L;
-    private final IPushRuntimeFactory[] runtimeFactories;
-    private final RecordDescriptor[] recordDescriptors;
-
-    public AlgebricksPipeline(IPushRuntimeFactory[] runtimeFactories, RecordDescriptor[] recordDescriptors) {
-        this.runtimeFactories = runtimeFactories;
-        this.recordDescriptors = recordDescriptors;
-        // this.projectedColumns = projectedColumns;
-    }
-
-    public IPushRuntimeFactory[] getRuntimeFactories() {
-        return runtimeFactories;
-    }
-
-    public RecordDescriptor[] getRecordDescriptors() {
-        return recordDescriptors;
-    }
-
-    public int getOutputWidth() {
-        return recordDescriptors[recordDescriptors.length - 1].getFieldCount();
-    }
-
-    // public int[] getProjectedColumns() {
-    // return projectedColumns;
-    // }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java
deleted file mode 100644
index 5719bff..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluator.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface IAggregateEvaluator {
-    public void init() throws AlgebricksException;
-
-    public void step(IFrameTupleReference tuple) throws AlgebricksException;
-
-    public void finishPartial(IPointable result) throws AlgebricksException;
-
-    public void finish(IPointable result) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java
deleted file mode 100644
index 1c8e1c7..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/IAggregateEvaluatorFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface IAggregateEvaluatorFactory extends Serializable {
-    public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java
deleted file mode 100644
index fd21e98..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunction.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface ICopyAggregateFunction {
-    /** should be called each time a new aggregate value is computed */
-    public void init() throws AlgebricksException;
-
-    public void step(IFrameTupleReference tuple) throws AlgebricksException;
-
-    public void finish() throws AlgebricksException;
-
-    public void finishPartial() throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java
deleted file mode 100644
index e8ae777..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyAggregateFunctionFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-
-public interface ICopyAggregateFunctionFactory extends Serializable {
-    public ICopyAggregateFunction createAggregateFunction(IDataOutputProvider provider) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java
deleted file mode 100644
index d593103..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluator.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface ICopyEvaluator {
-    public void evaluate(IFrameTupleReference tuple) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java
deleted file mode 100644
index beb8e47..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyEvaluatorFactory.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-
-public interface ICopyEvaluatorFactory extends Serializable {
-    public ICopyEvaluator createEvaluator(IDataOutputProvider output) throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java
deleted file mode 100644
index 8030425..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunction.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface ICopyRunningAggregateFunction {
-    public void init() throws AlgebricksException;
-
-    public void step(IFrameTupleReference tuple) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java
deleted file mode 100644
index 52982aa..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyRunningAggregateFunctionFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
-
-public interface ICopyRunningAggregateFunctionFactory extends Serializable {
-    public ICopyRunningAggregateFunction createRunningAggregateFunction(IDataOutputProvider provider)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java
deleted file mode 100644
index a9649b0..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunction.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.DataOutput;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface ICopySerializableAggregateFunction {
-    /**
-     * initialize the space occupied by internal state
-     * 
-     * @param state
-     * @throws AlgebricksException
-     * @return length of the intermediate state
-     */
-    public void init(DataOutput state) throws AlgebricksException;
-
-    /**
-     * update the internal state
-     * 
-     * @param tuple
-     * @param state
-     * @throws AlgebricksException
-     */
-    public void step(IFrameTupleReference tuple, byte[] data, int start, int len) throws AlgebricksException;
-
-    /**
-     * output the state to result
-     * 
-     * @param state
-     * @param result
-     * @throws AlgebricksException
-     */
-    public void finish(byte[] data, int start, int len, DataOutput result) throws AlgebricksException;
-
-    /**
-     * output the partial state to partial result
-     * 
-     * @param state
-     * @param partialResult
-     * @throws AlgebricksException
-     */
-    public void finishPartial(byte[] data, int start, int len, DataOutput partialResult) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java
deleted file mode 100644
index b813be0..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopySerializableAggregateFunctionFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public interface ICopySerializableAggregateFunctionFactory extends Serializable {
-    public ICopySerializableAggregateFunction createAggregateFunction() throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java b/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java
deleted file mode 100644
index d3a8a1b..0000000
--- a/algebricks/algebricks-runtime/src/main/java/edu/uci/ics/hyracks/algebricks/runtime/base/ICopyUnnestingFunction.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.runtime.base;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public interface ICopyUnnestingFunction {
-    public void init(IFrameTupleReference tuple) throws AlgebricksException;
-
-    public boolean step() throws AlgebricksException;
-
-}

[15/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java
deleted file mode 100644
index 1397956..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This rule eliminates a subplan with the following pattern:
- * -- SUBPLAN
- * -- OP (where OP produces exactly one tuple)
- * The live variables at OP will not be used after SUBPLAN.
- * Note: This rule must be applied after
- * the RemoveRedundantVariablesRule (to avoid the lineage analysis of variable cardinality).
- * 
- * @author yingyib
- */
-public class EliminateSubplanWithInputCardinalityOneRule implements IAlgebraicRewriteRule {
-    /** The pointer to the topmost operator */
-    private Mutable<ILogicalOperator> rootRef;
-    /** Whether the rule has even been invoked */
-    private boolean invoked = false;
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        if (!invoked) {
-            rootRef = opRef;
-            invoked = true;
-        }
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getInputs().size() <= 0) {
-            return false;
-        }
-        boolean changed = false;
-        for (Mutable<ILogicalOperator> subplanRef : op.getInputs()) {
-            AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRef.getValue();
-            if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-                continue;
-            }
-
-            SubplanOperator subplan = (SubplanOperator) op1;
-            Set<LogicalVariable> usedVarsUp = new ListSet<LogicalVariable>();
-            OperatorPropertiesUtil.getFreeVariablesInPath(rootRef.getValue(), subplan, usedVarsUp);
-            // TODO(buyingyi): figure out the rewriting for subplan operators with multiple subplans.
-            if (subplan.getNestedPlans().size() != 1) {
-                continue;
-            }
-
-            ILogicalOperator subplanInputOperator = subplan.getInputs().get(0).getValue();
-            Set<LogicalVariable> subplanInputVars = new ListSet<LogicalVariable>();
-            VariableUtilities.getLiveVariables(subplanInputOperator, subplanInputVars);
-            int subplanInputVarSize = subplanInputVars.size();
-            subplanInputVars.removeAll(usedVarsUp);
-            // Makes sure the free variables are only used in the subplan.
-            if (subplanInputVars.size() < subplanInputVarSize) {
-                continue;
-            }
-            Set<LogicalVariable> freeVars = new ListSet<LogicalVariable>();
-            OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
-            boolean cardinalityOne = isCardinalityOne(subplan.getInputs().get(0), freeVars);
-            if (cardinalityOne) {
-                /** If the cardinality of freeVars in the subplan is one, the subplan can be removed. */
-                ILogicalPlan plan = subplan.getNestedPlans().get(0);
-
-                List<Mutable<ILogicalOperator>> rootRefs = plan.getRoots();
-                // TODO(buyingyi): investigate the case of multi-root plans.
-                if (rootRefs.size() != 1) {
-                    continue;
-                }
-                Set<Mutable<ILogicalOperator>> ntsSet = new ListSet<Mutable<ILogicalOperator>>();
-                findNts(rootRefs.get(0), ntsSet);
-
-                /** Replaces nts with the input operator of the subplan. */
-                for (Mutable<ILogicalOperator> nts : ntsSet) {
-                    nts.setValue(subplanInputOperator);
-                }
-                subplanRef.setValue(rootRefs.get(0).getValue());
-                changed = true;
-            } else {
-                continue;
-            }
-        }
-        return changed;
-    }
-
-    /**
-     * Whether the cardinality of the input free variables are one.
-     * 
-     * @param opRef
-     *            the operator to be checked (including its input operators)
-     * @param freeVars
-     *            variables to be checked for produced operators
-     * @return true if every input variable has cardinality one; false otherwise.
-     * @throws AlgebricksException
-     */
-    private boolean isCardinalityOne(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> freeVars)
-            throws AlgebricksException {
-        Set<LogicalVariable> varsWithCardinalityOne = new ListSet<LogicalVariable>();
-        Set<LogicalVariable> varsLiveAtUnnestAndJoin = new ListSet<LogicalVariable>();
-        isCardinalityOne(opRef, freeVars, varsWithCardinalityOne, varsLiveAtUnnestAndJoin);
-        varsWithCardinalityOne.removeAll(varsLiveAtUnnestAndJoin);
-        return varsWithCardinalityOne.equals(freeVars);
-    }
-
-    /**
-     * Recursively adding variables which has cardinality one and in int the input free variable set.
-     * 
-     * @param opRef
-     *            , the current operator reference.
-     * @param freeVars
-     *            , a set of variables.
-     * @param varsWithCardinalityOne
-     *            , variables in the free variable set with cardinality one at the time they are created.
-     * @param varsLiveAtUnnestAndJoin
-     *            , live variables at Unnest and Join. The cardinalities of those variables can become more than one
-     *            even if their cardinalities were one at the time those variables were created.
-     * @throws AlgebricksException
-     */
-    private void isCardinalityOne(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> freeVars,
-            Set<LogicalVariable> varsWithCardinalityOne, Set<LogicalVariable> varsLiveAtUnnestAndJoin)
-            throws AlgebricksException {
-        AbstractLogicalOperator operator = (AbstractLogicalOperator) opRef.getValue();
-        List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
-        VariableUtilities.getProducedVariables(operator, producedVars);
-        if (operator.getOperatorTag() == LogicalOperatorTag.UNNEST
-                || operator.getOperatorTag() == LogicalOperatorTag.INNERJOIN
-                || operator.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
-            VariableUtilities.getLiveVariables(operator, varsLiveAtUnnestAndJoin);
-        }
-        if (operator.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
-            for (LogicalVariable producedVar : producedVars) {
-                if (freeVars.contains(producedVar)) {
-                    varsWithCardinalityOne.add(producedVar);
-                }
-            }
-        }
-        if (varsWithCardinalityOne.size() == freeVars.size()) {
-            return;
-        }
-        for (Mutable<ILogicalOperator> childRef : operator.getInputs()) {
-            isCardinalityOne(childRef, freeVars, varsWithCardinalityOne, varsLiveAtUnnestAndJoin);
-        }
-    }
-
-    /**
-     * Find the NestedTupleSource operator in the direct/undirect input operators of opRef.
-     * 
-     * @param opRef
-     *            , the current operator reference.
-     * @param ntsSet
-     *            , the set NestedTupleSource operator references.
-     */
-    private void findNts(Mutable<ILogicalOperator> opRef, Set<Mutable<ILogicalOperator>> ntsSet) {
-        int childSize = opRef.getValue().getInputs().size();
-        if (childSize == 0) {
-            AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-            if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-                ntsSet.add(opRef);
-            }
-            return;
-        }
-        for (Mutable<ILogicalOperator> childRef : opRef.getValue().getInputs()) {
-            findNts(childRef, ntsSet);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java
deleted file mode 100644
index d4834a3..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * If there is any ordering property before the subplan operator, the ordering should
- * be kept after the subplan.
- * This rule adds a redundant order operator after those cases, to guarantee the correctness.
- * 
- * @author yingyib
- */
-public class EnforceOrderByAfterSubplan implements IAlgebraicRewriteRule {
-    /** a set of order-breaking operators */
-    private final Set<LogicalOperatorTag> orderBreakingOps = new HashSet<LogicalOperatorTag>();
-    /** a set of order-sensitive operators */
-    private final Set<LogicalOperatorTag> orderSensitiveOps = new HashSet<LogicalOperatorTag>();
-
-    public EnforceOrderByAfterSubplan() {
-        /** add operators that break the ordering */
-        orderBreakingOps.add(LogicalOperatorTag.INNERJOIN);
-        orderBreakingOps.add(LogicalOperatorTag.LEFTOUTERJOIN);
-        orderBreakingOps.add(LogicalOperatorTag.UNIONALL);
-        orderBreakingOps.add(LogicalOperatorTag.AGGREGATE);
-
-        /** add operators that are sensitive to the ordering */
-        orderSensitiveOps.add(LogicalOperatorTag.LIMIT);
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (context.checkIfInDontApplySet(this, op1)) {
-            return false;
-        }
-        List<Mutable<ILogicalOperator>> inputs = op1.getInputs();
-        context.addToDontApplySet(this, op1);
-        if (op1.getOperatorTag() == LogicalOperatorTag.ORDER || inputs == null) {
-            /**
-             * does not apply if
-             * 1. there is yet-another order operator on-top-of the subplan, because the downstream order operator's ordering will be broken anyway
-             * 2. the input operator(s) is null
-             */
-            return false;
-        }
-        boolean changed = false;
-        for (int i = 0; i < inputs.size(); i++) {
-            Mutable<ILogicalOperator> inputOpRef = inputs.get(i);
-            AbstractLogicalOperator op = (AbstractLogicalOperator) inputOpRef.getValue();
-            context.addToDontApplySet(this, op);
-            if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-                continue;
-            }
-
-            /**
-             * check the order operators whose ordering is not broken before the subplan operator, and then
-             * duplicate them on-top-of the subplan operator
-             */
-            boolean foundTarget = true;
-            boolean orderSensitive = false;
-            Mutable<ILogicalOperator> childRef = op.getInputs().get(0);
-            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
-            while (child.getOperatorTag() != LogicalOperatorTag.ORDER) {
-                context.addToDontApplySet(this, child);
-                if (orderBreakingOps.contains(child.getOperatorTag())) {
-                    foundTarget = false;
-                    break;
-                }
-                if(child.getOperatorTag() == LogicalOperatorTag.GROUP){
-                    foundTarget = false;
-                    break;
-                }
-                if (orderSensitiveOps.contains(child.getOperatorTag())) {
-                    orderSensitive = true;
-                }
-                List<Mutable<ILogicalOperator>> childInputs = child.getInputs();
-                if (childInputs == null || childInputs.size() > 2 || childInputs.size() < 1) {
-                    foundTarget = false;
-                    break;
-                } else {
-                    childRef = childInputs.get(0);
-                    child = (AbstractLogicalOperator) childRef.getValue();
-                }
-            }
-            /** the target order-by operator has not been found. */
-            if (!foundTarget) {
-                return false;
-            }
-
-            /** copy the original order-by operator and insert on-top-of the subplan operator */
-            context.addToDontApplySet(this, child);
-            OrderOperator sourceOrderOp = (OrderOperator) child;
-            for (Pair<IOrder, Mutable<ILogicalExpression>> expr : sourceOrderOp.getOrderExpressions()) {
-                if (!expr.second.getValue().isFunctional()) {
-                    return false;
-                }
-            }
-            List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = deepCopyOrderAndExpression(sourceOrderOp
-                    .getOrderExpressions());
-            OrderOperator newOrderOp = new OrderOperator(orderExprs);
-            context.addToDontApplySet(this, newOrderOp);
-            inputs.set(i, new MutableObject<ILogicalOperator>(newOrderOp));
-            newOrderOp.getInputs().add(inputOpRef);
-            context.computeAndSetTypeEnvironmentForOperator(newOrderOp);
-
-            if (!orderSensitive) {
-                /** remove the original order-by */
-                childRef.setValue(sourceOrderOp.getInputs().get(0).getValue());
-            }
-            changed = true;
-        }
-        return changed;
-    }
-
-    private Mutable<ILogicalExpression> deepCopyExpressionRef(Mutable<ILogicalExpression> oldExpr) {
-        return new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression());
-    }
-
-    private List<Pair<IOrder, Mutable<ILogicalExpression>>> deepCopyOrderAndExpression(
-            List<Pair<IOrder, Mutable<ILogicalExpression>>> ordersAndExprs) {
-        List<Pair<IOrder, Mutable<ILogicalExpression>>> newOrdersAndExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
-        for (Pair<IOrder, Mutable<ILogicalExpression>> pair : ordersAndExprs)
-            newOrdersAndExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(pair.first,
-                    deepCopyExpressionRef(pair.second)));
-        return newOrdersAndExprs;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
deleted file mode 100644
index ebd7da1..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
+++ /dev/null
@@ -1,614 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractStableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.BroadcastPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryStableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreSortedDistinctByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RandomMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RandomPartitionPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionMergePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.DefaultNodeGroupDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty.PartitioningType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
-import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.range.IRangeMap;
-
-public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule {
-
-    private static final INodeDomain DEFAULT_DOMAIN = new DefaultNodeGroupDomain("__DEFAULT");
-
-    private PhysicalOptimizationConfig physicalOptimizationConfig;
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        // wait for the physical operators to be set first
-        if (op.getPhysicalOperator() == null) {
-            return false;
-        }
-        if (context.checkIfInDontApplySet(this, op)) {
-            return false;
-        }
-
-        List<FunctionalDependency> fds = context.getFDList(op);
-        if (fds != null && !fds.isEmpty()) {
-            return false;
-        }
-        // These are actually logical constraints, so they could be pre-computed
-        // somewhere else, too.
-
-        physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
-        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Optimizing operator " + op.getPhysicalOperator() + ".\n");
-
-        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
-
-        StructuralPropertiesVector pvector = new StructuralPropertiesVector(new RandomPartitioningProperty(null),
-                new LinkedList<ILocalStructuralProperty>());
-        boolean changed = physOptimizeOp(opRef, pvector, false, context);
-        op.computeDeliveredPhysicalProperties(context);
-        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator() + ": "
-                + op.getDeliveredPhysicalProperties() + "\n");
-
-        context.addToDontApplySet(this, opRef.getValue());
-
-        return changed;
-    }
-
-    private boolean physOptimizePlan(ILogicalPlan plan, IPhysicalPropertiesVector pvector, boolean nestedPlan,
-            IOptimizationContext context) throws AlgebricksException {
-        boolean changed = false;
-        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
-            if (physOptimizeOp(root, pvector, nestedPlan, context)) {
-                changed = true;
-            }
-            AbstractLogicalOperator op = (AbstractLogicalOperator) root.getValue();
-            op.computeDeliveredPhysicalProperties(context);
-            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator()
-                    + ": " + op.getDeliveredPhysicalProperties() + "\n");
-        }
-        return changed;
-    }
-
-    private boolean physOptimizeOp(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector required,
-            boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
-
-        boolean changed = false;
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        optimizeUsingConstraintsAndEquivClasses(op);
-        PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required);
-        IPhysicalPropertiesVector[] reqdProperties = null;
-        if (pr != null) {
-            reqdProperties = pr.getRequiredProperties();
-        }
-        boolean opIsRedundantSort = false;
-
-        // compute properties and figure out the domain
-        INodeDomain childrenDomain = null;
-        {
-            int j = 0;
-            for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
-                AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
-                // recursive call
-                if (physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context)) {
-                    changed = true;
-                }
-                child.computeDeliveredPhysicalProperties(context);
-                IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
-                if (childrenDomain == null) {
-                    childrenDomain = delivered.getPartitioningProperty().getNodeDomain();
-                } else {
-                    INodeDomain dom2 = delivered.getPartitioningProperty().getNodeDomain();
-                    if (!childrenDomain.sameAs(dom2)) {
-                        childrenDomain = DEFAULT_DOMAIN;
-                    }
-                }
-                j++;
-            }
-        }
-
-        if (reqdProperties != null) {
-            for (int k = 0; k < reqdProperties.length; k++) {
-                IPhysicalPropertiesVector pv = reqdProperties[k];
-                IPartitioningProperty pp = pv.getPartitioningProperty();
-                if (pp != null && pp.getNodeDomain() == null) {
-                    pp.setNodeDomain(childrenDomain);
-                }
-            }
-        }
-
-        IPartitioningProperty firstDeliveredPartitioning = null;
-        int i = 0;
-        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
-            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
-            IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
-
-            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Properties delivered by " + child.getPhysicalOperator()
-                    + ": " + delivered + "\n");
-            IPartitioningRequirementsCoordinator prc = pr.getPartitioningCoordinator();
-            Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(
-                    reqdProperties[i].getPartitioningProperty(), firstDeliveredPartitioning, op, context);
-            boolean mayExpandPartitioningProperties = pbpp.first;
-            IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second,
-                    reqdProperties[i].getLocalProperties());
-
-            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for " + child.getPhysicalOperator()
-                    + ": " + rqd + "\n");
-            IPhysicalPropertiesVector diff = delivered.getUnsatisfiedPropertiesFrom(rqd,
-                    mayExpandPartitioningProperties, context.getEquivalenceClassMap(child), context.getFDList(child));
-
-            if (isRedundantSort(opRef, delivered, diff, context)) {
-                opIsRedundantSort = true;
-            }
-
-            if (diff != null) {
-                changed = true;
-                addEnforcers(op, i, diff, rqd, delivered, childrenDomain, nestedPlan, context);
-
-                AbstractLogicalOperator newChild = ((AbstractLogicalOperator) op.getInputs().get(i).getValue());
-
-                if (newChild != child) {
-                    delivered = newChild.getDeliveredPhysicalProperties();
-                    IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, rqd,
-                            mayExpandPartitioningProperties, context);
-                    AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
-
-                    if (isRedundantSort(opRef, delivered, newDiff, context)) {
-                        opIsRedundantSort = true;
-                        break;
-                    }
-                }
-
-            }
-            if (firstDeliveredPartitioning == null) {
-                IPartitioningProperty dpp = delivered.getPartitioningProperty();
-                if (dpp.getPartitioningType() == PartitioningType.ORDERED_PARTITIONED
-                        || dpp.getPartitioningType() == PartitioningType.UNORDERED_PARTITIONED) {
-                    firstDeliveredPartitioning = dpp;
-                }
-            }
-
-            i++;
-        }
-
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan p : nested.getNestedPlans()) {
-                if (physOptimizePlan(p, required, true, context)) {
-                    changed = true;
-                }
-            }
-        }
-
-        if (opIsRedundantSort) {
-            if (AlgebricksConfig.DEBUG) {
-                AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Removing redundant SORT operator "
-                        + op.getPhysicalOperator() + "\n");
-                printOp(op);
-            }
-            changed = true;
-            AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-            if (nextOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-                nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getValue();
-            }
-            opRef.setValue(nextOp);
-            // Now, transfer annotations from the original sort op. to this one.
-            AbstractLogicalOperator transferTo = nextOp;
-            if (transferTo.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-                //
-                // remove duplicate exchange operator
-                transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getValue();
-            }
-            transferTo.getAnnotations().putAll(op.getAnnotations());
-            physOptimizeOp(opRef, required, nestedPlan, context);
-        }
-        return changed;
-    }
-
-    private IPhysicalPropertiesVector newPropertiesDiff(AbstractLogicalOperator newChild,
-            IPhysicalPropertiesVector required, boolean mayExpandPartitioningProperties, IOptimizationContext context)
-            throws AlgebricksException {
-        IPhysicalPropertiesVector newDelivered = newChild.getDeliveredPhysicalProperties();
-
-        Map<LogicalVariable, EquivalenceClass> newChildEqClasses = context.getEquivalenceClassMap(newChild);
-        List<FunctionalDependency> newChildFDs = context.getFDList(newChild);
-        if (newChildEqClasses == null || newChildFDs == null) {
-            FDsAndEquivClassesVisitor fdsVisitor = new FDsAndEquivClassesVisitor();
-            newChild.accept(fdsVisitor, context);
-            newChildEqClasses = context.getEquivalenceClassMap(newChild);
-            newChildFDs = context.getFDList(newChild);
-        }
-        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for new op. "
-                + newChild.getPhysicalOperator() + ": " + required + "\n");
-
-        return newDelivered.getUnsatisfiedPropertiesFrom(required, mayExpandPartitioningProperties, newChildEqClasses,
-                newChildFDs);
-    }
-
-    private void optimizeUsingConstraintsAndEquivClasses(AbstractLogicalOperator op) {
-        IPhysicalOperator pOp = op.getPhysicalOperator();
-        switch (pOp.getOperatorTag()) {
-            case HASH_GROUP_BY:
-            case EXTERNAL_GROUP_BY: {
-                GroupByOperator gby = (GroupByOperator) op;
-                ExternalGroupByPOperator hgbyOp = (ExternalGroupByPOperator) pOp;
-                hgbyOp.computeColumnSet(gby.getGroupByList());
-                break;
-            }
-            case PRE_CLUSTERED_GROUP_BY: {
-                GroupByOperator gby = (GroupByOperator) op;
-                PreclusteredGroupByPOperator preSortedGby = (PreclusteredGroupByPOperator) pOp;
-                preSortedGby.setGbyColumns(gby.getGbyVarList());
-                break;
-            }
-            case PRE_SORTED_DISTINCT_BY: {
-                DistinctOperator d = (DistinctOperator) op;
-                PreSortedDistinctByPOperator preSortedDistinct = (PreSortedDistinctByPOperator) pOp;
-                preSortedDistinct.setDistinctByColumns(d.getDistinctByVarList());
-                break;
-            }
-        }
-    }
-
-    private List<OrderColumn> getOrderColumnsFromGroupingProperties(List<ILocalStructuralProperty> reqd,
-            List<ILocalStructuralProperty> dlvd) {
-        List<OrderColumn> returnedProperties = new ArrayList<OrderColumn>();
-        List<LogicalVariable> rqdCols = new ArrayList<LogicalVariable>();
-        List<LogicalVariable> dlvdCols = new ArrayList<LogicalVariable>();
-        for (ILocalStructuralProperty r : reqd) {
-            r.getVariables(rqdCols);
-        }
-        for (ILocalStructuralProperty d : dlvd) {
-            d.getVariables(dlvdCols);
-        }
-
-        int prefix = dlvdCols.size() - 1;
-        for (; prefix >= 0;) {
-            if (!rqdCols.contains(dlvdCols.get(prefix))) {
-                prefix--;
-            } else {
-                break;
-            }
-        }
-
-        LocalOrderProperty orderProp = (LocalOrderProperty) dlvd.get(0);
-        List<OrderColumn> orderColumns = orderProp.getOrderColumns();
-        for (int j = 0; j <= prefix; j++) {
-            returnedProperties.add(new OrderColumn(orderColumns.get(j).getColumn(), orderColumns.get(j).getOrder()));
-        }
-        // maintain other order columns after the required order columns
-        if (returnedProperties.size() != 0) {
-            for (int j = prefix + 1; j < dlvdCols.size(); j++) {
-                OrderColumn oc = orderColumns.get(j);
-                returnedProperties.add(new OrderColumn(oc.getColumn(), oc.getOrder()));
-            }
-        }
-        return returnedProperties;
-    }
-
-    /*
-     * We assume delivered to be already normalized.
-     */
-    private boolean isRedundantSort(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector delivered,
-            IPhysicalPropertiesVector diffOfProperties, IOptimizationContext context) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.ORDER
-                || (op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && op
-                        .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT)
-                || delivered.getLocalProperties() == null) {
-            return false;
-        }
-        AbstractStableSortPOperator sortOp = (AbstractStableSortPOperator) op.getPhysicalOperator();
-        sortOp.computeLocalProperties(op);
-        ILocalStructuralProperty orderProp = sortOp.getOrderProperty();
-        return PropertiesUtil.matchLocalProperties(Collections.singletonList(orderProp),
-                delivered.getLocalProperties(), context.getEquivalenceClassMap(op), context.getFDList(op));
-    }
-
-    private void addEnforcers(AbstractLogicalOperator op, int childIndex,
-            IPhysicalPropertiesVector diffPropertiesVector, IPhysicalPropertiesVector required,
-            IPhysicalPropertiesVector deliveredByChild, INodeDomain domain, boolean nestedPlan,
-            IOptimizationContext context) throws AlgebricksException {
-
-        IPartitioningProperty pp = diffPropertiesVector.getPartitioningProperty();
-        if (pp == null || pp.getPartitioningType() == PartitioningType.UNPARTITIONED) {
-            addLocalEnforcers(op, childIndex, diffPropertiesVector.getLocalProperties(), nestedPlan, context);
-            IPhysicalPropertiesVector deliveredByNewChild = ((AbstractLogicalOperator) op.getInputs().get(0).getValue())
-                    .getDeliveredPhysicalProperties();
-            addPartitioningEnforcers(op, childIndex, pp, required, deliveredByNewChild, domain, context);
-        } else {
-            addPartitioningEnforcers(op, childIndex, pp, required, deliveredByChild, domain, context);
-            AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
-            IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, required, true, context);
-            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
-            if (newDiff != null) {
-                addLocalEnforcers(op, childIndex, newDiff.getLocalProperties(), nestedPlan, context);
-            }
-        }
-    }
-
-    private void addLocalEnforcers(AbstractLogicalOperator op, int i, List<ILocalStructuralProperty> localProperties,
-            boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
-        if (AlgebricksConfig.DEBUG) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Adding local enforcers for local props = " + localProperties
-                    + "\n");
-        }
-
-        if (localProperties == null || localProperties.isEmpty()) {
-            return;
-        }
-
-        Mutable<ILogicalOperator> topOp = new MutableObject<ILogicalOperator>();
-        topOp.setValue(op.getInputs().get(i).getValue());
-        LinkedList<LocalOrderProperty> oList = new LinkedList<LocalOrderProperty>();
-
-        for (ILocalStructuralProperty prop : localProperties) {
-            switch (prop.getPropertyType()) {
-                case LOCAL_ORDER_PROPERTY: {
-                    oList.add((LocalOrderProperty) prop);
-                    break;
-                }
-                case LOCAL_GROUPING_PROPERTY: {
-                    LocalGroupingProperty g = (LocalGroupingProperty) prop;
-                    Collection<LogicalVariable> vars = (g.getPreferredOrderEnforcer() != null) ? g
-                            .getPreferredOrderEnforcer() : g.getColumnSet();
-                    List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
-                    for (LogicalVariable v : vars) {
-                        OrderColumn oc = new OrderColumn(v, OrderKind.ASC);
-                        orderColumns.add(oc);
-                    }
-                    LocalOrderProperty lop = new LocalOrderProperty(orderColumns);
-                    oList.add(lop);
-                    break;
-                }
-                default: {
-                    throw new IllegalStateException();
-                }
-            }
-        }
-        if (!oList.isEmpty()) {
-            topOp = enforceOrderProperties(oList, topOp, nestedPlan, context);
-        }
-
-        op.getInputs().set(i, topOp);
-        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) topOp.getValue(), context);
-        printOp((AbstractLogicalOperator) topOp.getValue());
-    }
-
-    private Mutable<ILogicalOperator> enforceOrderProperties(List<LocalOrderProperty> oList,
-            Mutable<ILogicalOperator> topOp, boolean isMicroOp, IOptimizationContext context)
-            throws AlgebricksException {
-        List<Pair<IOrder, Mutable<ILogicalExpression>>> oe = new LinkedList<Pair<IOrder, Mutable<ILogicalExpression>>>();
-        for (LocalOrderProperty orderProperty : oList) {
-            for (OrderColumn oc : orderProperty.getOrderColumns()) {
-                IOrder ordType = (oc.getOrder() == OrderKind.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
-                Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<IOrder, Mutable<ILogicalExpression>>(ordType,
-                        new MutableObject<ILogicalExpression>(new VariableReferenceExpression(oc.getColumn())));
-                oe.add(pair);
-            }
-        }
-        OrderOperator oo = new OrderOperator(oe);
-        oo.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
-        if (isMicroOp) {
-            oo.setPhysicalOperator(new InMemoryStableSortPOperator());
-        } else {
-            oo.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort()));
-        }
-        oo.getInputs().add(topOp);
-        context.computeAndSetTypeEnvironmentForOperator(oo);
-        if (AlgebricksConfig.DEBUG) {
-            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added sort enforcer " + oo.getPhysicalOperator() + ".\n");
-        }
-        return new MutableObject<ILogicalOperator>(oo);
-    }
-
-    private void addPartitioningEnforcers(ILogicalOperator op, int i, IPartitioningProperty pp,
-            IPhysicalPropertiesVector required, IPhysicalPropertiesVector deliveredByChild, INodeDomain domain,
-            IOptimizationContext context) throws AlgebricksException {
-        if (pp != null) {
-            IPhysicalOperator pop;
-            switch (pp.getPartitioningType()) {
-                case UNPARTITIONED: {
-                    List<OrderColumn> ordCols = computeOrderColumns(deliveredByChild);
-                    if (ordCols == null || ordCols.size() == 0) {
-                        pop = new RandomMergeExchangePOperator();
-                    } else {
-                        if (op.getAnnotations().containsKey(OperatorAnnotations.USE_RANGE_CONNECTOR)) {
-                            IRangeMap rangeMap = (IRangeMap) op.getAnnotations().get(
-                                    OperatorAnnotations.USE_RANGE_CONNECTOR);
-                            pop = new RangePartitionMergePOperator(ordCols, domain, rangeMap);
-                        } else {
-                            OrderColumn[] sortColumns = new OrderColumn[ordCols.size()];
-                            sortColumns = ordCols.toArray(sortColumns);
-                            pop = new SortMergeExchangePOperator(sortColumns);
-                        }
-                    }
-                    break;
-                }
-                case UNORDERED_PARTITIONED: {
-                    List<LogicalVariable> varList = new ArrayList<LogicalVariable>(
-                            ((UnorderedPartitionedProperty) pp).getColumnSet());
-                    List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties();
-                    List<ILocalStructuralProperty> reqdLocals = required.getLocalProperties();
-                    boolean propWasSet = false;
-                    pop = null;
-                    if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) {
-                        AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
-                        Map<LogicalVariable, EquivalenceClass> ecs = context.getEquivalenceClassMap(c);
-                        List<FunctionalDependency> fds = context.getFDList(c);
-                        if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) {
-                            List<OrderColumn> orderColumns = getOrderColumnsFromGroupingProperties(reqdLocals,
-                                    cldLocals);
-                            pop = new HashPartitionMergeExchangePOperator(orderColumns, varList, domain);
-                            propWasSet = true;
-                        }
-                    }
-                    if (!propWasSet) {
-                        pop = new HashPartitionExchangePOperator(varList, domain);
-                    }
-                    break;
-                }
-                case ORDERED_PARTITIONED: {
-                    pop = new RangePartitionPOperator(((OrderedPartitionedProperty) pp).getOrderColumns(), domain, null);
-                    break;
-                }
-                case BROADCAST: {
-                    pop = new BroadcastPOperator(domain);
-                    break;
-                }
-                case RANDOM: {
-                    RandomPartitioningProperty rpp = (RandomPartitioningProperty) pp;
-                    INodeDomain nd = rpp.getNodeDomain();
-                    pop = new RandomPartitionPOperator(nd);
-                    break;
-                }
-                default: {
-                    throw new NotImplementedException("Enforcer for " + pp.getPartitioningType()
-                            + " partitioning type has not been implemented.");
-                }
-            }
-            Mutable<ILogicalOperator> ci = op.getInputs().get(i);
-            ExchangeOperator exchg = new ExchangeOperator();
-            exchg.setPhysicalOperator(pop);
-            setNewOp(ci, exchg, context);
-            exchg.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
-            OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(exchg, context);
-            context.computeAndSetTypeEnvironmentForOperator(exchg);
-            if (AlgebricksConfig.DEBUG) {
-                AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added partitioning enforcer "
-                        + exchg.getPhysicalOperator() + ".\n");
-                printOp((AbstractLogicalOperator) op);
-            }
-        }
-    }
-
-    private boolean allAreOrderProps(List<ILocalStructuralProperty> cldLocals) {
-        for (ILocalStructuralProperty lsp : cldLocals) {
-            if (lsp.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
-                return false;
-            }
-        }
-        return !cldLocals.isEmpty();
-    }
-
-    private void printOp(AbstractLogicalOperator op) throws AlgebricksException {
-        StringBuilder sb = new StringBuilder();
-        LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
-        PlanPrettyPrinter.printOperator(op, sb, pvisitor, 0);
-        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(sb.toString());
-    }
-
-    private List<OrderColumn> computeOrderColumns(IPhysicalPropertiesVector pv) {
-        List<OrderColumn> ordCols = new ArrayList<OrderColumn>();
-        List<ILocalStructuralProperty> localProps = pv.getLocalProperties();
-        if (localProps == null || localProps.size() == 0) {
-            return null;
-        } else {
-            for (ILocalStructuralProperty p : localProps) {
-                if (p.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
-                    LocalOrderProperty lop = (LocalOrderProperty) p;
-                    ordCols.addAll(lop.getOrderColumns());
-                } else {
-                    return null;
-                }
-            }
-            return ordCols;
-        }
-
-    }
-
-    private void setNewOp(Mutable<ILogicalOperator> opRef, AbstractLogicalOperator newOp, IOptimizationContext context)
-            throws AlgebricksException {
-        ILogicalOperator oldOp = opRef.getValue();
-        opRef.setValue(newOp);
-        newOp.getInputs().add(new MutableObject<ILogicalOperator>(oldOp));
-        newOp.recomputeSchema();
-        newOp.computeDeliveredPhysicalProperties(context);
-        context.computeAndSetTypeEnvironmentForOperator(newOp);
-        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + newOp.getPhysicalOperator()
-                + ": " + newOp.getDeliveredPhysicalProperties() + "\n");
-
-        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(newOp, context);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
deleted file mode 100644
index 41cf470..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonExpressionsRule.java
+++ /dev/null
@@ -1,436 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Factors out common sub-expressions by assigning them to a variables, and replacing the common sub-expressions with references to those variables.
- * Preconditions/Assumptions:
- * Assumes no projects are in the plan. This rule ignores variable reference expressions and constants (other rules deal with those separately).
- * Postconditions/Examples:
- * Plan with extracted sub-expressions. Generates one assign operator per extracted expression.
- * Example 1 - Simple Arithmetic Example (simplified)
- * Before plan:
- * assign [$$1] <- [5 + 6 - 10]
- * assign [$$0] <- [5 + 6 + 30]
- * After plan:
- * assign [$$1] <- [$$5 - 10]
- * assign [$$0] <- [$$5 + 30]
- * assign [$$5] <- [5 + 6]
- * Example 2 - Cleaning up 'Distinct By' (simplified)
- * Before plan: (notice how $$0 is not live after the distinct)
- * assign [$$3] <- [field-access($$0, 1)]
- * distinct ([%0->$$5])
- * assign [$$5] <- [field-access($$0, 1)]
- * unnest $$0 <- [scan-dataset]
- * After plan: (notice how the issue of $$0 is fixed)
- * assign [$$3] <- [$$5]
- * distinct ([$$5])
- * assign [$$5] <- [field-access($$0, 1)]
- * unnest $$0 <- [scan-dataset]
- * Example 3 - Pulling Common Expressions Above Joins (simplified)
- * Before plan:
- * assign [$$9] <- funcZ(funcY($$8))
- * join (funcX(funcY($$8)))
- * After plan:
- * assign [$$9] <- funcZ($$10))
- * select (funcX($$10))
- * assign [$$10] <- [funcY($$8)]
- * join (TRUE)
- */
-public class ExtractCommonExpressionsRule implements IAlgebraicRewriteRule {
-
-    private final List<ILogicalExpression> originalAssignExprs = new ArrayList<ILogicalExpression>();
-
-    private final CommonExpressionSubstitutionVisitor substVisitor = new CommonExpressionSubstitutionVisitor();
-    private final Map<ILogicalExpression, ExprEquivalenceClass> exprEqClassMap = new HashMap<ILogicalExpression, ExprEquivalenceClass>();
-
-    // Set of operators for which common subexpression elimination should not be performed.
-    private static final Set<LogicalOperatorTag> ignoreOps = new HashSet<LogicalOperatorTag>();
-    static {
-        ignoreOps.add(LogicalOperatorTag.UNNEST);
-        ignoreOps.add(LogicalOperatorTag.UNNEST_MAP);
-        ignoreOps.add(LogicalOperatorTag.ORDER);
-        ignoreOps.add(LogicalOperatorTag.PROJECT);
-        ignoreOps.add(LogicalOperatorTag.AGGREGATE);
-        ignoreOps.add(LogicalOperatorTag.RUNNINGAGGREGATE);
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        exprEqClassMap.clear();
-        substVisitor.setContext(context);
-        boolean modified = removeCommonExpressions(opRef, context);
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(opRef.getValue());
-        }
-        return modified;
-    }
-
-    private void updateEquivalenceClassMap(LogicalVariable lhs, Mutable<ILogicalExpression> rhsExprRef,
-            ILogicalExpression rhsExpr, ILogicalOperator op) {
-        ExprEquivalenceClass exprEqClass = exprEqClassMap.get(rhsExpr);
-        if (exprEqClass == null) {
-            exprEqClass = new ExprEquivalenceClass(op, rhsExprRef);
-            exprEqClassMap.put(rhsExpr, exprEqClass);
-        }
-        exprEqClass.setVariable(lhs);
-    }
-
-    private boolean removeCommonExpressions(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
-            return false;
-        }
-
-        boolean modified = false;
-        // Recurse into children.
-        for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
-            if (removeCommonExpressions(inputOpRef, context)) {
-                modified = true;
-            }
-        }
-
-        // TODO: Deal with replicate properly. Currently, we just clear the expr equivalence map, since we want to avoid incorrect expression replacement
-        // (the resulting new variables should be assigned live below a replicate).
-        if (op.getOperatorTag() == LogicalOperatorTag.REPLICATE) {
-            exprEqClassMap.clear();
-            return modified;
-        }
-        // Exclude these operators.
-        if (ignoreOps.contains(op.getOperatorTag())) {
-            return modified;
-        }
-
-        // Remember a copy of the original assign expressions, so we can add them to the equivalence class map
-        // after replacing expressions within the assign operator itself.
-        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-            AssignOperator assignOp = (AssignOperator) op;
-            originalAssignExprs.clear();
-            int numVars = assignOp.getVariables().size();
-            for (int i = 0; i < numVars; i++) {
-                Mutable<ILogicalExpression> exprRef = assignOp.getExpressions().get(i);
-                ILogicalExpression expr = exprRef.getValue();
-                originalAssignExprs.add(expr.cloneExpression());
-            }
-        }
-
-        // Perform common subexpression elimination.
-        substVisitor.setOperator(op);
-        if (op.acceptExpressionTransform(substVisitor)) {
-            modified = true;
-        }
-
-        // Update equivalence class map.
-        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-            AssignOperator assignOp = (AssignOperator) op;
-            int numVars = assignOp.getVariables().size();
-            for (int i = 0; i < numVars; i++) {
-                Mutable<ILogicalExpression> exprRef = assignOp.getExpressions().get(i);
-                ILogicalExpression expr = exprRef.getValue();
-                if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE
-                        || expr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
-                    continue;
-                }
-                // Update equivalence class map.
-                LogicalVariable lhs = assignOp.getVariables().get(i);
-                updateEquivalenceClassMap(lhs, exprRef, exprRef.getValue(), op);
-
-                // Update equivalence class map with original assign expression.
-                updateEquivalenceClassMap(lhs, exprRef, originalAssignExprs.get(i), op);
-            }
-        }
-
-        // TODO: For now do not perform replacement in nested plans
-        // due to the complication of figuring out whether the firstOp in an equivalence class is within a subplan, 
-        // and the resulting variable will not be visible to the outside.
-        // Since subplans should be eliminated in most cases, this behavior is acceptable for now.
-        /*
-        if (op.hasNestedPlans()) {
-            AbstractOperatorWithNestedPlans opWithNestedPlan = (AbstractOperatorWithNestedPlans) op;
-            for (ILogicalPlan nestedPlan : opWithNestedPlan.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> rootRef : nestedPlan.getRoots()) {
-                    if (removeCommonExpressions(rootRef, context)) {
-                        modified = true;
-                    }
-                }
-            }
-        }
-        */
-
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(op);
-            context.addToDontApplySet(this, op);
-        }
-        return modified;
-    }
-
-    private class CommonExpressionSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
-
-        private IOptimizationContext context;
-        private ILogicalOperator op;
-
-        public void setContext(IOptimizationContext context) {
-            this.context = context;
-        }
-
-        public void setOperator(ILogicalOperator op) throws AlgebricksException {
-            this.op = op;
-        }
-
-        @Override
-        public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
-            AbstractLogicalExpression expr = (AbstractLogicalExpression) exprRef.getValue();
-            boolean modified = false;
-            ExprEquivalenceClass exprEqClass = exprEqClassMap.get(expr);
-            if (exprEqClass != null) {
-                // Replace common subexpression with existing variable. 
-                if (exprEqClass.variableIsSet()) {
-                    Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
-                    List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
-                    VariableUtilities.getLiveVariables(op, liveVars);
-                    VariableUtilities.getUsedVariables(op, usedVars);
-                    // Check if the replacing variable is live at this op.
-                    // However, if the op is already using variables that are not live, then a replacement may enable fixing the plan.
-                    // This behavior is necessary to, e.g., properly deal with distinct by.
-                    // Also just replace the expr if we are replacing common exprs from within the same operator.
-                    if (liveVars.contains(exprEqClass.getVariable()) || !liveVars.containsAll(usedVars)
-                            || op == exprEqClass.getFirstOperator()) {
-                        exprRef.setValue(new VariableReferenceExpression(exprEqClass.getVariable()));
-                        // Do not descend into children since this expr has been completely replaced.
-                        return true;
-                    }
-                } else {
-                    if (expr.isFunctional() && assignCommonExpression(exprEqClass, expr)) {
-                        //re-obtain the live vars after rewriting in the method called in the if condition
-                        Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
-                        VariableUtilities.getLiveVariables(op, liveVars);
-                        //rewrite only when the variable is live
-                        if (liveVars.contains(exprEqClass.getVariable())) {
-                            exprRef.setValue(new VariableReferenceExpression(exprEqClass.getVariable()));
-                            // Do not descend into children since this expr has been completely replaced.
-                            return true;
-                        }
-                    }
-                }
-            } else {
-                if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE
-                        && expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
-                    exprEqClass = new ExprEquivalenceClass(op, exprRef);
-                    exprEqClassMap.put(expr, exprEqClass);
-                }
-            }
-
-            // Descend into function arguments.
-            if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-                for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
-                    if (transform(arg)) {
-                        modified = true;
-                    }
-                }
-            }
-            return modified;
-        }
-
-        private boolean assignCommonExpression(ExprEquivalenceClass exprEqClass, ILogicalExpression expr)
-                throws AlgebricksException {
-            AbstractLogicalOperator firstOp = (AbstractLogicalOperator) exprEqClass.getFirstOperator();
-            Mutable<ILogicalExpression> firstExprRef = exprEqClass.getFirstExpression();
-            if (firstOp.getOperatorTag() == LogicalOperatorTag.INNERJOIN
-                    || firstOp.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
-                // Do not extract common expressions from within the same join operator.
-                if (firstOp == op) {
-                    return false;
-                }
-                AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) firstOp;
-                Mutable<ILogicalExpression> joinCond = joinOp.getCondition();
-                ILogicalExpression enclosingExpr = getEnclosingExpression(joinCond, firstExprRef.getValue());
-                if (enclosingExpr == null) {
-                    // No viable enclosing expression that we can pull out from the join.
-                    return false;
-                }
-                // Place a Select operator beneath op that contains the enclosing expression.
-                SelectOperator selectOp = new SelectOperator(new MutableObject<ILogicalExpression>(enclosingExpr),
-                        false, null);
-                selectOp.getInputs().add(new MutableObject<ILogicalOperator>(op.getInputs().get(0).getValue()));
-                op.getInputs().get(0).setValue(selectOp);
-                // Set firstOp to be the select below op, since we want to assign the common subexpr there.
-                firstOp = (AbstractLogicalOperator) selectOp;
-            } else if (firstOp.getInputs().size() > 1) {
-                // Bail for any non-join operator with multiple inputs.
-                return false;
-            }
-            LogicalVariable newVar = context.newVar();
-            AssignOperator newAssign = new AssignOperator(newVar, new MutableObject<ILogicalExpression>(firstExprRef
-                    .getValue().cloneExpression()));
-            // Place assign below firstOp.
-            newAssign.getInputs().add(new MutableObject<ILogicalOperator>(firstOp.getInputs().get(0).getValue()));
-            newAssign.setExecutionMode(firstOp.getExecutionMode());
-            firstOp.getInputs().get(0).setValue(newAssign);
-            // Replace original expr with variable reference, and set var in expression equivalence class.
-            firstExprRef.setValue(new VariableReferenceExpression(newVar));
-            exprEqClass.setVariable(newVar);
-            context.computeAndSetTypeEnvironmentForOperator(newAssign);
-            context.computeAndSetTypeEnvironmentForOperator(firstOp);
-            return true;
-        }
-
-        private ILogicalExpression getEnclosingExpression(Mutable<ILogicalExpression> conditionExprRef,
-                ILogicalExpression commonSubExpr) {
-            ILogicalExpression conditionExpr = conditionExprRef.getValue();
-            if (conditionExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-                return null;
-            }
-            if (isEqJoinCondition(commonSubExpr)) {
-                // Do not eliminate the common expression if we could use it for an equi-join.
-                return null;
-            }
-            AbstractFunctionCallExpression conditionFuncExpr = (AbstractFunctionCallExpression) conditionExpr;
-            // Boolean expression that encloses the common subexpression.
-            ILogicalExpression enclosingBoolExpr = null;
-            // We are not dealing with arbitrarily nested and/or expressions here.
-            FunctionIdentifier funcIdent = conditionFuncExpr.getFunctionIdentifier();
-            if (funcIdent.equals(AlgebricksBuiltinFunctions.AND) || funcIdent.equals(AlgebricksBuiltinFunctions.OR)) {
-                Iterator<Mutable<ILogicalExpression>> argIter = conditionFuncExpr.getArguments().iterator();
-                while (argIter.hasNext()) {
-                    Mutable<ILogicalExpression> argRef = argIter.next();
-                    if (containsExpr(argRef.getValue(), commonSubExpr)) {
-                        enclosingBoolExpr = argRef.getValue();
-                        // Remove the enclosing expression from the argument list.
-                        // We are going to pull it out into a new select operator.
-                        argIter.remove();
-                        break;
-                    }
-                }
-                // If and/or only has a single argument left, pull it out and remove the and/or function.
-                if (conditionFuncExpr.getArguments().size() == 1) {
-                    conditionExprRef.setValue(conditionFuncExpr.getArguments().get(0).getValue());
-                }
-            } else {
-                if (!containsExpr(conditionExprRef.getValue(), commonSubExpr)) {
-                    return null;
-                }
-                enclosingBoolExpr = conditionFuncExpr;
-                // Replace the enclosing expression with TRUE.
-                conditionExprRef.setValue(ConstantExpression.TRUE);
-            }
-            return enclosingBoolExpr;
-        }
-    }
-
-    private boolean containsExpr(ILogicalExpression expr, ILogicalExpression searchExpr) {
-        if (expr == searchExpr) {
-            return true;
-        }
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-        for (Mutable<ILogicalExpression> argRef : funcExpr.getArguments()) {
-            if (containsExpr(argRef.getValue(), searchExpr)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private boolean isEqJoinCondition(ILogicalExpression expr) {
-        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-        if (funcExpr.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.EQ)) {
-            ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
-            ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
-            if (arg1.getExpressionTag() == LogicalExpressionTag.VARIABLE
-                    && arg2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private final class ExprEquivalenceClass {
-        // First operator in which expression is used.
-        private final ILogicalOperator firstOp;
-
-        // Reference to expression in first op.
-        private final Mutable<ILogicalExpression> firstExprRef;
-
-        // Variable that this expression has been assigned to.
-        private LogicalVariable var;
-
-        public ExprEquivalenceClass(ILogicalOperator firstOp, Mutable<ILogicalExpression> firstExprRef) {
-            this.firstOp = firstOp;
-            this.firstExprRef = firstExprRef;
-        }
-
-        public ILogicalOperator getFirstOperator() {
-            return firstOp;
-        }
-
-        public Mutable<ILogicalExpression> getFirstExpression() {
-            return firstExprRef;
-        }
-
-        public void setVariable(LogicalVariable var) {
-            this.var = var;
-        }
-
-        public LogicalVariable getVariable() {
-            return var;
-        }
-
-        public boolean variableIsSet() {
-            return var != null;
-        }
-    }
-}

[25/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
new file mode 100644
index 0000000..a24c48d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
+
+public class PreSortedDistinctByPOperator extends AbstractPhysicalOperator {
+
+    private List<LogicalVariable> columnList;
+
+    public PreSortedDistinctByPOperator(List<LogicalVariable> columnList) {
+        this.columnList = columnList;
+    }
+
+    public void setDistinctByColumns(List<LogicalVariable> distinctByColumns) {
+        this.columnList = distinctByColumns;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
+        List<ILocalStructuralProperty> propsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
+        deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+        List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>();
+        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+        for (LogicalVariable column : columnList) {
+            orderColumns.add(new OrderColumn(column, OrderKind.ASC));
+        }
+        localProps.add(new LocalOrderProperty(orderColumns));
+        IPartitioningProperty pp = null;
+        AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
+        if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
+            pp = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(columnList), null);
+        }
+        pv[0] = new StructuralPropertiesVector(pp, localProps);
+        return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
+        int sz = inputSchemas[0].getSize();
+        int fdSz = sz - columnList.size();
+        int[] fdColumns = new int[fdSz];
+        int j = 0;
+        for (LogicalVariable v : inputSchemas[0]) {
+            if (!columnList.contains(v)) {
+                fdColumns[j++] = inputSchemas[0].findVariable(v);
+            }
+        }
+        int[] keysAndDecs = new int[keys.length + fdColumns.length];
+        for (int i = 0; i < keys.length; i++) {
+            keysAndDecs[i] = keys[i];
+        }
+        for (int i = 0; i < fdColumns.length; i++) {
+            keysAndDecs[i + keys.length] = fdColumns[i];
+        }
+
+        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
+                columnList, context.getTypeEnvironment(op), context);
+        IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {};
+        IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(
+                aggFactories, keysAndDecs);
+
+        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema,
+                context);
+        /** make fd columns part of the key but the comparator only compares the distinct key columns */
+        PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keysAndDecs,
+                comparatorFactories, aggregatorFactory, recordDescriptor);
+
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
new file mode 100644
index 0000000..bcd998d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.NestedPlansRunningAggregatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.preclustered.PreclusteredGroupOperatorDescriptor;
+
+public class PreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
+
+    public PreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
+        super(columnList);
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
+        GroupByOperator gby = (GroupByOperator) op;
+        int numFds = gby.getDecorList().size();
+        int fdColumns[] = new int[numFds];
+        int j = 0;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new AlgebricksException("pre-sorted group-by expects variable references.");
+            }
+            VariableReferenceExpression v = (VariableReferenceExpression) expr;
+            LogicalVariable decor = v.getVariableReference();
+            fdColumns[j++] = inputSchemas[0].findVariable(decor);
+        }
+        // compile subplans and set the gby op. schema accordingly
+        AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
+        IAggregatorDescriptorFactory aggregatorFactory;
+
+        if (((AbstractLogicalOperator) (gby.getNestedPlans().get(0).getRoots().get(0).getValue())).getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) {
+            aggregatorFactory = new NestedPlansRunningAggregatorFactory(subplans, keys, fdColumns);
+        } else {
+            aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns);
+        }
+
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
+                columnList, context.getTypeEnvironment(op), context);
+        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema,
+                context);
+
+        PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys,
+                comparatorFactories, aggregatorFactory, recordDescriptor);
+
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
new file mode 100644
index 0000000..2478859
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
+
+public class RandomMergeExchangePOperator extends AbstractExchangePOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.RANDOM_MERGE_EXCHANGE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        this.deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
+                new ArrayList<ILocalStructuralProperty>(0));
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
+        IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java
new file mode 100644
index 0000000..c633f7b
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RandomPartitionPOperator.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.RandomPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
+
+public class RandomPartitionPOperator extends AbstractExchangePOperator {
+
+    private final INodeDomain domain;
+
+    public RandomPartitionPOperator(INodeDomain domain) {
+        this.domain = domain;
+    }
+
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op,
+                opSchema, context);
+        builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        ITuplePartitionComputerFactory tpcf = new RandomPartitionComputerFactory(domain.cardinality());
+        MToNPartitioningConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.RANDOM_PARTITION_EXCHANGE;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        this.deliveredProperties = new StructuralPropertiesVector(new RandomPartitioningProperty(domain), op2
+                .getDeliveredPhysicalProperties().getLocalProperties());
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java
new file mode 100644
index 0000000..99f68e9
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionMergePOperator.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.range.FieldRangePartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.range.IRangeMap;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class RangePartitionMergePOperator extends AbstractExchangePOperator {
+
+    private List<OrderColumn> partitioningFields;
+    private INodeDomain domain;
+    private IRangeMap rangeMap;
+
+    public RangePartitionMergePOperator(List<OrderColumn> partitioningFields, INodeDomain domain, IRangeMap rangeMap) {
+        this.partitioningFields = partitioningFields;
+        this.domain = domain;
+        this.rangeMap = rangeMap;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.RANGE_PARTITION_MERGE_EXCHANGE;
+    }
+
+    public List<OrderColumn> getPartitioningFields() {
+        return partitioningFields;
+    }
+
+    public INodeDomain getDomain() {
+        return domain;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        List<LogicalVariable> varList = new ArrayList<LogicalVariable>();
+        for (OrderColumn oc : partitioningFields) {
+            varList.add(oc.getColumn());
+        }
+        IPartitioningProperty p = new UnorderedPartitionedProperty(new ListSet<LogicalVariable>(varList), domain);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
+        List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
+        for (ILocalStructuralProperty prop : op2Locals) {
+            if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+                locals.add(prop);
+            } else {
+                break;
+            }
+        }
+
+        this.deliveredProperties = new StructuralPropertiesVector(p, locals);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<ILocalStructuralProperty> orderProps = new LinkedList<ILocalStructuralProperty>();
+        List<OrderColumn> columns = new ArrayList<OrderColumn>();
+        for (OrderColumn oc : partitioningFields) {
+            LogicalVariable var = oc.getColumn();
+            columns.add(new OrderColumn(var, oc.getOrder()));
+        }
+        orderProps.add(new LocalOrderProperty(columns));
+        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
+                orderProps) };
+        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        int n = partitioningFields.size();
+        int[] sortFields = new int[n];
+        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        INormalizedKeyComputerFactory nkcf = null;
+
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        int i = 0;
+        for (OrderColumn oc : partitioningFields) {
+            LogicalVariable var = oc.getColumn();
+            sortFields[i] = opSchema.findVariable(var);
+            Object type = env.getVarType(var);
+            OrderKind order = oc.getOrder();
+            if (i == 0 && nkcfProvider != null && type != null) {
+                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
+            }
+            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
+            i++;
+        }
+        ITuplePartitionComputerFactory tpcf = new FieldRangePartitionComputerFactory(sortFields, comps, rangeMap);
+        IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " " + partitioningFields + " SPLIT COUNT:" + rangeMap.getSplitCount();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
new file mode 100644
index 0000000..66955e9
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.range.FieldRangePartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.range.IRangeMap;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
+
+public class RangePartitionPOperator extends AbstractExchangePOperator {
+
+    private List<OrderColumn> partitioningFields;
+    private INodeDomain domain;
+    private IRangeMap rangeMap;
+
+    public RangePartitionPOperator(List<OrderColumn> partitioningFields, INodeDomain domain, IRangeMap rangeMap) {
+        this.partitioningFields = partitioningFields;
+        this.domain = domain;
+        this.rangeMap = rangeMap;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.RANGE_PARTITION_EXCHANGE;
+    }
+
+    public List<OrderColumn> getPartitioningFields() {
+        return partitioningFields;
+    }
+
+    public INodeDomain getDomain() {
+        return domain;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        IPartitioningProperty p = new OrderedPartitionedProperty(new ArrayList<OrderColumn>(partitioningFields), domain);
+        this.deliveredProperties = new StructuralPropertiesVector(p, new LinkedList<ILocalStructuralProperty>());
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        int n = partitioningFields.size();
+        int[] sortFields = new int[n];
+        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        INormalizedKeyComputerFactory nkcf = null;
+
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        int i = 0;
+        for (OrderColumn oc : partitioningFields) {
+            LogicalVariable var = oc.getColumn();
+            sortFields[i] = opSchema.findVariable(var);
+            Object type = env.getVarType(var);
+            OrderKind order = oc.getOrder();
+            if (i == 0 && nkcfProvider != null && type != null) {
+                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
+            }
+            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
+            i++;
+        }
+        ITuplePartitionComputerFactory tpcf = new FieldRangePartitionComputerFactory(sortFields, comps, rangeMap);
+        IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " " + partitioningFields + " SPLIT COUNT:" + rangeMap.getSplitCount();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
new file mode 100644
index 0000000..d8f7d99
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.misc.SplitOperatorDescriptor;
+
+public class ReplicatePOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.SPLIT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
+
+        ReplicateOperator rop = (ReplicateOperator) op;
+        int outputArity = rop.getOutputArity();
+        boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags();
+
+        SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity, outputMaterializationFlags);
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, splitOpDesc);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[] { 0 };
+        ReplicateOperator rop = (ReplicateOperator) op;
+        int[] outputDependencyLabels = new int[rop.getOutputArity()];
+        // change the labels of outputs that requires materialization to 1
+        boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags();
+        for (int i = 0; i < rop.getOutputArity(); i++) {
+            if (outputMaterializationFlags[i]) {
+                outputDependencyLabels[i] = 1;
+            }
+        }
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
new file mode 100644
index 0000000..3b2387a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.RunningAggregateRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class RunningAggregatePOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.RUNNING_AGGREGATE;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+        List<LogicalVariable> variables = ragg.getVariables();
+        List<Mutable<ILogicalExpression>> expressions = ragg.getExpressions();
+        int[] outColumns = new int[variables.size()];
+        for (int i = 0; i < outColumns.length; i++) {
+            outColumns[i] = opSchema.findVariable(variables.get(i));
+        }
+        IRunningAggregateEvaluatorFactory[] runningAggFuns = new IRunningAggregateEvaluatorFactory[expressions.size()];
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        for (int i = 0; i < runningAggFuns.length; i++) {
+            StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getValue();
+            runningAggFuns[i] = expressionRuntimeProvider.createRunningAggregateFunctionFactory(expr,
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+        }
+
+        // TODO push projections into the operator
+        int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
+
+        RunningAggregateRuntimeFactory runtime = new RunningAggregateRuntimeFactory(outColumns, runningAggFuns,
+                projectionList);
+
+        // contribute one Asterix framewriter
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        builder.contributeMicroOperator(ragg, runtime, recDesc);
+        // and contribute one edge from its child
+        ILogicalOperator src = ragg.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, ragg, 0);
+
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
new file mode 100644
index 0000000..ba299ac
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.base.SinkRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.misc.SinkOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
+
+public class SinkPOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.SINK;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2;
+        List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
+        IPhysicalPropertiesVector childsProperties = null;
+
+        for (int i = 0; i < op.getInputs().size(); i++) {
+            op2 = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
+            childsProperties = op2.getPhysicalOperator().getDeliveredProperties();
+            if (childsProperties.getLocalProperties() != null) {
+                propsLocal.addAll(childsProperties.getLocalProperties());
+            }
+        }
+
+        deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+      return emptyUnaryRequirements(op.getInputs().size());
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+
+        SinkOperatorDescriptor opDesc = new SinkOperatorDescriptor(spec, op.getInputs().size());
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+        for (int i = 0; i < op.getInputs().size(); i++) {
+            builder.contributeGraphEdge(op.getInputs().get(i).getValue(), 0, op, i);
+        }
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
new file mode 100644
index 0000000..9105c1f
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class SinkWritePOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.SINK_WRITE;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        WriteOperator write = (WriteOperator) op;
+        IDataSink sink = write.getDataSink();
+        IPartitioningProperty pp = sink.getPartitioningProperty();
+        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp, null) };
+        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        WriteOperator write = (WriteOperator) op;
+        int[] columns = new int[write.getExpressions().size()];
+        int i = 0;
+        for (Mutable<ILogicalExpression> exprRef : write.getExpressions()) {
+            ILogicalExpression expr = exprRef.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new NotImplementedException("Only writing variable expressions is supported.");
+            }
+            VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+            LogicalVariable v = varRef.getVariableReference();
+            columns[i++] = inputSchemas[0].findVariable(v);
+        }
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context);
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0],
+                context);
+
+        IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
+                context, columns);
+
+        IMetadataProvider<?, ?> mp = context.getMetadataProvider();
+
+        Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> runtime = mp.getWriteFileRuntime(write.getDataSink(),
+                columns, pf, inputDesc);
+
+        builder.contributeMicroOperator(write, runtime.first, recDesc, runtime.second);
+        ILogicalOperator src = write.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, write, 0);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java
new file mode 100644
index 0000000..6ad1fab
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortGroupByPOperator.java
@@ -0,0 +1,291 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.OperatorSchemaImpl;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.sort.SortGroupByOperatorDescriptor;
+
+public class SortGroupByPOperator extends AbstractPhysicalOperator {
+
+    private final int frameLimit;
+    private final OrderColumn[] orderColumns;
+    private final List<LogicalVariable> columnSet = new ArrayList<LogicalVariable>();
+
+    public SortGroupByPOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList, int frameLimit,
+            OrderColumn[] orderColumns) {
+        this.frameLimit = frameLimit;
+        this.orderColumns = orderColumns;
+        computeColumnSet(gbyList);
+    }
+
+    private void computeColumnSet(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList) {
+        columnSet.clear();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                VariableReferenceExpression v = (VariableReferenceExpression) expr;
+                columnSet.add(v.getVariableReference());
+            }
+        }
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.SORT_GROUP_BY;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + columnSet;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    private List<LogicalVariable> getGbyColumns() {
+        return columnSet;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
+
+        GroupByOperator gOp = (GroupByOperator) op;
+        Set<LogicalVariable> columnSet = new ListSet<LogicalVariable>();
+        List<OrderColumn> ocs = new ArrayList<OrderColumn>();
+
+        if (!columnSet.isEmpty()) {
+            propsLocal.add(new LocalGroupingProperty(columnSet));
+        }
+        for (OrderColumn oc : orderColumns) {
+            ocs.add(oc);
+        }
+        propsLocal.add(new LocalOrderProperty(ocs));
+        for (ILogicalPlan p : gOp.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator rOp = r.getValue();
+                propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
+            }
+        }
+
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
+        deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        List<LogicalVariable> gbyCols = getGbyColumns();
+        int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]);
+        GroupByOperator gby = (GroupByOperator) op;
+        int numFds = gby.getDecorList().size();
+        int fdColumns[] = new int[numFds];
+        int j = 0;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                throw new AlgebricksException("Sort group-by expects variable references.");
+            }
+            VariableReferenceExpression v = (VariableReferenceExpression) expr;
+            LogicalVariable decor = v.getVariableReference();
+            fdColumns[j++] = inputSchemas[0].findVariable(decor);
+        }
+
+        if (gby.getNestedPlans().size() != 1) {
+            throw new AlgebricksException(
+                    "Sort group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        ILogicalPlan p0 = gby.getNestedPlans().get(0);
+        if (p0.getRoots().size() != 1) {
+            throw new AlgebricksException(
+                    "Sort group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+
+        IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
+        List<Object> intermediateTypes = new ArrayList<Object>();
+        int n = aggOp.getExpressions().size();
+        IAggregateEvaluatorFactory[] aff = new IAggregateEvaluatorFactory[n];
+        int i = 0;
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
+        IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
+        for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
+            aff[i++] = expressionRuntimeProvider.createAggregateFunctionFactory(aggFun, aggOpInputEnv, inputSchemas,
+                    context);
+            intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
+                    context.getMetadataProvider()));
+        }
+
+        int[] keyAndDecFields = new int[keys.length + fdColumns.length];
+        for (i = 0; i < keys.length; ++i) {
+            keyAndDecFields[i] = keys[i];
+        }
+        for (i = 0; i < fdColumns.length; i++) {
+            keyAndDecFields[keys.length + i] = fdColumns[i];
+        }
+
+        List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList()) {
+            keyAndDecVariables.add(p.first);
+        }
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
+        }
+
+        for (LogicalVariable var : keyAndDecVariables) {
+            aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
+        }
+
+        compileSubplans(inputSchemas[0], gby, opSchema, context);
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+
+        IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[gbyCols.size()];
+        IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
+        i = 0;
+        for (LogicalVariable v : gbyCols) {
+            Object type = aggOpInputEnv.getVarType(v);
+            if (orderColumns[i].getOrder() == OrderKind.ASC) {
+                compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true);
+            } else {
+                compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, false);
+            }
+            i++;
+        }
+        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema,
+                context);
+
+        IAggregateEvaluatorFactory[] merges = new IAggregateEvaluatorFactory[n];
+        List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+        IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
+        localInputSchemas[0] = new OperatorSchemaImpl();
+        for (i = 0; i < n; i++) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
+                    .get(i).getValue();
+            aggFun.getUsedVariables(usedVars);
+        }
+        i = 0;
+        for (Object type : intermediateTypes) {
+            aggOpInputEnv.setVarType(usedVars.get(i++), type);
+        }
+        for (LogicalVariable keyVar : keyAndDecVariables) {
+            localInputSchemas[0].addVariable(keyVar);
+        }
+        for (LogicalVariable usedVar : usedVars) {
+            localInputSchemas[0].addVariable(usedVar);
+        }
+        for (i = 0; i < n; i++) {
+            AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
+                    .get(i).getValue();
+            merges[i] = expressionRuntimeProvider.createAggregateFunctionFactory(mergeFun, aggOpInputEnv,
+                    localInputSchemas, context);
+        }
+        RecordDescriptor partialAggRecordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op),
+                localInputSchemas[0], context);
+
+        IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aff,
+                keyAndDecFields);
+        IAggregatorDescriptorFactory mergeFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(merges,
+                keyAndDecFields);
+
+        INormalizedKeyComputerFactory normalizedKeyFactory = null;
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        if (nkcfProvider == null) {
+            normalizedKeyFactory = null;
+        }
+        Object type = aggOpInputEnv.getVarType(gbyCols.get(0));
+        normalizedKeyFactory = orderColumns[0].getOrder() == OrderKind.ASC ? nkcfProvider
+                .getNormalizedKeyComputerFactory(type, true) : nkcfProvider
+                .getNormalizedKeyComputerFactory(type, false);
+        SortGroupByOperatorDescriptor gbyOpDesc = new SortGroupByOperatorDescriptor(spec, frameLimit, keys,
+                keyAndDecFields, normalizedKeyFactory, compFactories, aggregatorFactory, mergeFactory,
+                partialAggRecordDescriptor, recordDescriptor, false);
+
+        contributeOpDesc(builder, gby, gbyOpDesc);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public Pair<int[], int[]> getInputOutputDependencyLabels(ILogicalOperator op) {
+        int[] inputDependencyLabels = new int[] { 0 };
+        int[] outputDependencyLabels = new int[] { 1 };
+        return new Pair<int[], int[]>(inputDependencyLabels, outputDependencyLabels);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
new file mode 100644
index 0000000..7f29375
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.IConnectorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class SortMergeExchangePOperator extends AbstractExchangePOperator {
+
+    private final OrderColumn[] sortColumns;
+
+    public SortMergeExchangePOperator(OrderColumn[] sortColumns) {
+        this.sortColumns = sortColumns;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.SORT_MERGE_EXCHANGE;
+    }
+
+    public OrderColumn[] getSortColumns() {
+        return sortColumns;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        sb.append(getOperatorTag());
+        sb.append(" [");
+        sb.append(sortColumns[0]);
+        for (int i = 1; i < sortColumns.length; i++) {
+            sb.append(", " + sortColumns[i]);
+        }
+        sb.append(" ]");
+        return sb.toString();
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator inp1 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        IPhysicalPropertiesVector pv1 = inp1.getDeliveredPhysicalProperties();
+        if (pv1 == null) {
+            inp1.computeDeliveredPhysicalProperties(context);
+            pv1 = inp1.getDeliveredPhysicalProperties();
+        }
+
+        List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+        List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(sortColumns.length);
+        for (ILocalStructuralProperty prop : pv1.getLocalProperties()) {
+            if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+                LocalOrderProperty lop = (LocalOrderProperty) prop;
+                for (OrderColumn oc : lop.getOrderColumns()) {
+                    if (oc.equals(sortColumns[orderColumns.size()])) {
+                        orderColumns.add(oc);
+                        if (orderColumns.size() == sortColumns.length) {
+                            break;
+                        }
+                    } else {
+                        break;
+                    }
+                }
+            } else {
+                continue;
+            }
+        }
+        if (orderColumns.size() > 0) {
+            localProps.add(new LocalOrderProperty(orderColumns));
+        }
+        this.deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, localProps);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(sortColumns.length);
+        localProps.add(new LocalOrderProperty(Arrays.asList(sortColumns)));
+        StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
+                localProps) };
+        return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(IConnectorDescriptorRegistry spec,
+            ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+        int n = sortColumns.length;
+        int[] sortFields = new int[n];
+        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+        IBinaryHashFunctionFactory[] hashFuns = new IBinaryHashFunctionFactory[n];
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        INormalizedKeyComputerFactory nkcf = null;
+
+        for (int i = 0; i < n; i++) {
+            sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn());
+            Object type = env.getVarType(sortColumns[i].getColumn());
+            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+            comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
+            IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider();
+            hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type);
+            if (i == 0 && nkcfProvider != null && type != null) {
+                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, sortColumns[i].getOrder() == OrderKind.ASC);
+            }
+        }
+        ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(sortFields, hashFuns);
+        IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps, nkcf);
+        return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
+    }
+
+}

[09/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java
new file mode 100644
index 0000000..2f080fb
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/CopyLimitDownRule.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class CopyLimitDownRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.LIMIT) {
+            return false;
+        }
+        LimitOperator limitOp = (LimitOperator) op;
+        if (!limitOp.isTopmostLimitOp()) {
+            return false;
+        }
+
+        List<LogicalVariable> limitUsedVars = new ArrayList<>();
+        VariableUtilities.getUsedVariables(limitOp, limitUsedVars);
+
+        Mutable<ILogicalOperator> safeOpRef = null;
+        Mutable<ILogicalOperator> candidateOpRef = limitOp.getInputs().get(0);
+
+        List<LogicalVariable> candidateProducedVars = new ArrayList<>();
+        while (true) {
+            candidateProducedVars.clear();
+            ILogicalOperator candidateOp = candidateOpRef.getValue();
+            LogicalOperatorTag candidateOpTag = candidateOp.getOperatorTag();
+            if (candidateOp.getInputs().size() > 1 || !candidateOp.isMap()
+                    || candidateOpTag == LogicalOperatorTag.SELECT || candidateOpTag == LogicalOperatorTag.LIMIT
+                    || !OperatorPropertiesUtil.disjoint(limitUsedVars, candidateProducedVars)) {
+                break;
+            }
+
+            safeOpRef = candidateOpRef;
+            candidateOpRef = safeOpRef.getValue().getInputs().get(0);
+        }
+
+        if (safeOpRef != null) {
+            ILogicalOperator safeOp = safeOpRef.getValue();
+            Mutable<ILogicalOperator> unsafeOpRef = safeOp.getInputs().get(0);
+            ILogicalOperator unsafeOp = unsafeOpRef.getValue();
+            LimitOperator limitCloneOp = null;
+            if (limitOp.getOffset().getValue() == null) {
+                limitCloneOp = new LimitOperator(limitOp.getMaxObjects().getValue(), false);
+            } else {
+                IFunctionInfo finfoAdd = context.getMetadataProvider().lookupFunction(
+                        AlgebricksBuiltinFunctions.NUMERIC_ADD);
+                List<Mutable<ILogicalExpression>> addArgs = new ArrayList<>();
+                addArgs.add(new MutableObject<ILogicalExpression>(limitOp.getMaxObjects().getValue().cloneExpression()));
+                addArgs.add(new MutableObject<ILogicalExpression>(limitOp.getOffset().getValue().cloneExpression()));
+                ScalarFunctionCallExpression maxPlusOffset = new ScalarFunctionCallExpression(finfoAdd, addArgs);
+                limitCloneOp = new LimitOperator(maxPlusOffset, false);
+            }
+            limitCloneOp.setPhysicalOperator(new StreamLimitPOperator());
+            limitCloneOp.getInputs().add(new MutableObject<ILogicalOperator>(unsafeOp));
+            limitCloneOp.setExecutionMode(unsafeOp.getExecutionMode());
+            limitCloneOp.recomputeSchema();
+            unsafeOpRef.setValue(limitCloneOp);
+            context.computeAndSetTypeEnvironmentForOperator(limitCloneOp);
+            context.addToDontApplySet(this, limitOp);
+        }
+
+        return safeOpRef != null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java
new file mode 100644
index 0000000..e93fdd1
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateGroupByEmptyKeyRule.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule lift out the aggregate operator out from a group-by operator
+ * if the gby operator groups-by on empty key, e.g., the group-by variables are empty.
+ * 
+ * @author yingyib
+ */
+public class EliminateGroupByEmptyKeyRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        GroupByOperator groupOp = (GroupByOperator) op;
+        List<LogicalVariable> groupVars = groupOp.getGbyVarList();
+        if (groupVars.size() > 0) {
+            return false;
+        }
+        List<ILogicalPlan> nestedPlans = groupOp.getNestedPlans();
+        if (nestedPlans.size() > 1) {
+            return false;
+        }
+        ILogicalPlan nestedPlan = nestedPlans.get(0);
+        if (nestedPlan.getRoots().size() > 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> topOpRef = nestedPlan.getRoots().get(0);
+        ILogicalOperator topOp = nestedPlan.getRoots().get(0).getValue();
+        Mutable<ILogicalOperator> nestedTupleSourceRef = getNestedTupleSourceReference(topOpRef);
+        /**
+         * connect nested top op into the plan
+         */
+        opRef.setValue(topOp);
+        /**
+         * connect child op into the plan
+         */
+        nestedTupleSourceRef.setValue(groupOp.getInputs().get(0).getValue());
+        return true;
+    }
+
+    private Mutable<ILogicalOperator> getNestedTupleSourceReference(Mutable<ILogicalOperator> nestedTopOperatorRef) {
+        Mutable<ILogicalOperator> currentOpRef = nestedTopOperatorRef;
+        while (currentOpRef.getValue().getInputs() != null && currentOpRef.getValue().getInputs().size() > 0) {
+            currentOpRef = currentOpRef.getValue().getInputs().get(0);
+        }
+        return currentOpRef;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
new file mode 100644
index 0000000..8a381f7
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.LinkedList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class EliminateSubplanRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    /**
+     * Eliminate Subplan above ETS
+     * and Subplan that has only ops. with one input and no free vars. (could we
+     * modify it to consider free vars which are sources of Unnest or Assign, if
+     * there are no aggregates?)
+     */
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op;
+
+        Mutable<ILogicalOperator> outerRef = subplan.getInputs().get(0);
+        AbstractLogicalOperator outerRefOp = (AbstractLogicalOperator) outerRef.getValue();
+        if (outerRefOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+            elimSubplanOverEts(opRef, context);
+            return true;
+        }
+        if (subplan.getNestedPlans().size() == 1 && subplan.getNestedPlans().get(0).getRoots().size() == 1
+                && !OperatorPropertiesUtil.hasFreeVariables(subplan)) {
+            if (elimOneSubplanWithNoFreeVars(opRef)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private boolean elimOneSubplanWithNoFreeVars(Mutable<ILogicalOperator> opRef) {
+        SubplanOperator subplan = (SubplanOperator) opRef.getValue();
+        AbstractLogicalOperator rootOp = (AbstractLogicalOperator) subplan.getNestedPlans().get(0).getRoots().get(0)
+                .getValue();
+        if (rootOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
+                || rootOp.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            opRef.setValue(subplan.getInputs().get(0).getValue());
+            return true;
+        } else {
+            AbstractLogicalOperator botOp = rootOp;
+            if (botOp.getInputs().size() != 1) {
+                return false;
+            }
+            do {
+                Mutable<ILogicalOperator> botRef = botOp.getInputs().get(0);
+                botOp = (AbstractLogicalOperator) botRef.getValue();
+                if (botOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+                    botRef.setValue(subplan.getInputs().get(0).getValue());
+                    opRef.setValue(rootOp);
+                    return true;
+                }
+            } while (botOp.getInputs().size() == 1);
+            return false;
+        }
+    }
+
+    private void elimSubplanOverEts(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx)
+            throws AlgebricksException {
+        SubplanOperator subplan = (SubplanOperator) opRef.getValue();
+        for (ILogicalPlan p : subplan.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.ntsToEts(r, ctx);
+            }
+        }
+        LinkedList<Mutable<ILogicalOperator>> allRoots = subplan.allRootsInReverseOrder();
+        if (allRoots.size() == 1) {
+            opRef.setValue(allRoots.get(0).getValue());
+        } else {
+            ILogicalOperator topOp = null;
+            for (Mutable<ILogicalOperator> r : allRoots) {
+                if (topOp == null) {
+                    topOp = r.getValue();
+                } else {
+                    LeftOuterJoinOperator j = new LeftOuterJoinOperator(new MutableObject<ILogicalExpression>(
+                            ConstantExpression.TRUE));
+                    j.getInputs().add(new MutableObject<ILogicalOperator>(topOp));
+                    j.getInputs().add(r);
+                    ctx.setOutputTypeEnvironment(j, j.computeOutputTypeEnvironment(ctx));
+                    topOp = j;
+                }
+            }
+            opRef.setValue(topOp);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java
new file mode 100644
index 0000000..1397956
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EliminateSubplanWithInputCardinalityOneRule.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule eliminates a subplan with the following pattern:
+ * -- SUBPLAN
+ * -- OP (where OP produces exactly one tuple)
+ * The live variables at OP will not be used after SUBPLAN.
+ * Note: This rule must be applied after
+ * the RemoveRedundantVariablesRule (to avoid the lineage analysis of variable cardinality).
+ * 
+ * @author yingyib
+ */
+public class EliminateSubplanWithInputCardinalityOneRule implements IAlgebraicRewriteRule {
+    /** The pointer to the topmost operator */
+    private Mutable<ILogicalOperator> rootRef;
+    /** Whether the rule has even been invoked */
+    private boolean invoked = false;
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (!invoked) {
+            rootRef = opRef;
+            invoked = true;
+        }
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getInputs().size() <= 0) {
+            return false;
+        }
+        boolean changed = false;
+        for (Mutable<ILogicalOperator> subplanRef : op.getInputs()) {
+            AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRef.getValue();
+            if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+                continue;
+            }
+
+            SubplanOperator subplan = (SubplanOperator) op1;
+            Set<LogicalVariable> usedVarsUp = new ListSet<LogicalVariable>();
+            OperatorPropertiesUtil.getFreeVariablesInPath(rootRef.getValue(), subplan, usedVarsUp);
+            // TODO(buyingyi): figure out the rewriting for subplan operators with multiple subplans.
+            if (subplan.getNestedPlans().size() != 1) {
+                continue;
+            }
+
+            ILogicalOperator subplanInputOperator = subplan.getInputs().get(0).getValue();
+            Set<LogicalVariable> subplanInputVars = new ListSet<LogicalVariable>();
+            VariableUtilities.getLiveVariables(subplanInputOperator, subplanInputVars);
+            int subplanInputVarSize = subplanInputVars.size();
+            subplanInputVars.removeAll(usedVarsUp);
+            // Makes sure the free variables are only used in the subplan.
+            if (subplanInputVars.size() < subplanInputVarSize) {
+                continue;
+            }
+            Set<LogicalVariable> freeVars = new ListSet<LogicalVariable>();
+            OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
+            boolean cardinalityOne = isCardinalityOne(subplan.getInputs().get(0), freeVars);
+            if (cardinalityOne) {
+                /** If the cardinality of freeVars in the subplan is one, the subplan can be removed. */
+                ILogicalPlan plan = subplan.getNestedPlans().get(0);
+
+                List<Mutable<ILogicalOperator>> rootRefs = plan.getRoots();
+                // TODO(buyingyi): investigate the case of multi-root plans.
+                if (rootRefs.size() != 1) {
+                    continue;
+                }
+                Set<Mutable<ILogicalOperator>> ntsSet = new ListSet<Mutable<ILogicalOperator>>();
+                findNts(rootRefs.get(0), ntsSet);
+
+                /** Replaces nts with the input operator of the subplan. */
+                for (Mutable<ILogicalOperator> nts : ntsSet) {
+                    nts.setValue(subplanInputOperator);
+                }
+                subplanRef.setValue(rootRefs.get(0).getValue());
+                changed = true;
+            } else {
+                continue;
+            }
+        }
+        return changed;
+    }
+
+    /**
+     * Whether the cardinality of the input free variables are one.
+     * 
+     * @param opRef
+     *            the operator to be checked (including its input operators)
+     * @param freeVars
+     *            variables to be checked for produced operators
+     * @return true if every input variable has cardinality one; false otherwise.
+     * @throws AlgebricksException
+     */
+    private boolean isCardinalityOne(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> freeVars)
+            throws AlgebricksException {
+        Set<LogicalVariable> varsWithCardinalityOne = new ListSet<LogicalVariable>();
+        Set<LogicalVariable> varsLiveAtUnnestAndJoin = new ListSet<LogicalVariable>();
+        isCardinalityOne(opRef, freeVars, varsWithCardinalityOne, varsLiveAtUnnestAndJoin);
+        varsWithCardinalityOne.removeAll(varsLiveAtUnnestAndJoin);
+        return varsWithCardinalityOne.equals(freeVars);
+    }
+
+    /**
+     * Recursively adding variables which has cardinality one and in int the input free variable set.
+     * 
+     * @param opRef
+     *            , the current operator reference.
+     * @param freeVars
+     *            , a set of variables.
+     * @param varsWithCardinalityOne
+     *            , variables in the free variable set with cardinality one at the time they are created.
+     * @param varsLiveAtUnnestAndJoin
+     *            , live variables at Unnest and Join. The cardinalities of those variables can become more than one
+     *            even if their cardinalities were one at the time those variables were created.
+     * @throws AlgebricksException
+     */
+    private void isCardinalityOne(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> freeVars,
+            Set<LogicalVariable> varsWithCardinalityOne, Set<LogicalVariable> varsLiveAtUnnestAndJoin)
+            throws AlgebricksException {
+        AbstractLogicalOperator operator = (AbstractLogicalOperator) opRef.getValue();
+        List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(operator, producedVars);
+        if (operator.getOperatorTag() == LogicalOperatorTag.UNNEST
+                || operator.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+                || operator.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+            VariableUtilities.getLiveVariables(operator, varsLiveAtUnnestAndJoin);
+        }
+        if (operator.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+            for (LogicalVariable producedVar : producedVars) {
+                if (freeVars.contains(producedVar)) {
+                    varsWithCardinalityOne.add(producedVar);
+                }
+            }
+        }
+        if (varsWithCardinalityOne.size() == freeVars.size()) {
+            return;
+        }
+        for (Mutable<ILogicalOperator> childRef : operator.getInputs()) {
+            isCardinalityOne(childRef, freeVars, varsWithCardinalityOne, varsLiveAtUnnestAndJoin);
+        }
+    }
+
+    /**
+     * Find the NestedTupleSource operator in the direct/undirect input operators of opRef.
+     * 
+     * @param opRef
+     *            , the current operator reference.
+     * @param ntsSet
+     *            , the set NestedTupleSource operator references.
+     */
+    private void findNts(Mutable<ILogicalOperator> opRef, Set<Mutable<ILogicalOperator>> ntsSet) {
+        int childSize = opRef.getValue().getInputs().size();
+        if (childSize == 0) {
+            AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+            if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                ntsSet.add(opRef);
+            }
+            return;
+        }
+        for (Mutable<ILogicalOperator> childRef : opRef.getValue().getInputs()) {
+            findNts(childRef, ntsSet);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java
new file mode 100644
index 0000000..d4834a3
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceOrderByAfterSubplan.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * If there is any ordering property before the subplan operator, the ordering should
+ * be kept after the subplan.
+ * This rule adds a redundant order operator after those cases, to guarantee the correctness.
+ * 
+ * @author yingyib
+ */
+public class EnforceOrderByAfterSubplan implements IAlgebraicRewriteRule {
+    /** a set of order-breaking operators */
+    private final Set<LogicalOperatorTag> orderBreakingOps = new HashSet<LogicalOperatorTag>();
+    /** a set of order-sensitive operators */
+    private final Set<LogicalOperatorTag> orderSensitiveOps = new HashSet<LogicalOperatorTag>();
+
+    public EnforceOrderByAfterSubplan() {
+        /** add operators that break the ordering */
+        orderBreakingOps.add(LogicalOperatorTag.INNERJOIN);
+        orderBreakingOps.add(LogicalOperatorTag.LEFTOUTERJOIN);
+        orderBreakingOps.add(LogicalOperatorTag.UNIONALL);
+        orderBreakingOps.add(LogicalOperatorTag.AGGREGATE);
+
+        /** add operators that are sensitive to the ordering */
+        orderSensitiveOps.add(LogicalOperatorTag.LIMIT);
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (context.checkIfInDontApplySet(this, op1)) {
+            return false;
+        }
+        List<Mutable<ILogicalOperator>> inputs = op1.getInputs();
+        context.addToDontApplySet(this, op1);
+        if (op1.getOperatorTag() == LogicalOperatorTag.ORDER || inputs == null) {
+            /**
+             * does not apply if
+             * 1. there is yet-another order operator on-top-of the subplan, because the downstream order operator's ordering will be broken anyway
+             * 2. the input operator(s) is null
+             */
+            return false;
+        }
+        boolean changed = false;
+        for (int i = 0; i < inputs.size(); i++) {
+            Mutable<ILogicalOperator> inputOpRef = inputs.get(i);
+            AbstractLogicalOperator op = (AbstractLogicalOperator) inputOpRef.getValue();
+            context.addToDontApplySet(this, op);
+            if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+                continue;
+            }
+
+            /**
+             * check the order operators whose ordering is not broken before the subplan operator, and then
+             * duplicate them on-top-of the subplan operator
+             */
+            boolean foundTarget = true;
+            boolean orderSensitive = false;
+            Mutable<ILogicalOperator> childRef = op.getInputs().get(0);
+            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
+            while (child.getOperatorTag() != LogicalOperatorTag.ORDER) {
+                context.addToDontApplySet(this, child);
+                if (orderBreakingOps.contains(child.getOperatorTag())) {
+                    foundTarget = false;
+                    break;
+                }
+                if(child.getOperatorTag() == LogicalOperatorTag.GROUP){
+                    foundTarget = false;
+                    break;
+                }
+                if (orderSensitiveOps.contains(child.getOperatorTag())) {
+                    orderSensitive = true;
+                }
+                List<Mutable<ILogicalOperator>> childInputs = child.getInputs();
+                if (childInputs == null || childInputs.size() > 2 || childInputs.size() < 1) {
+                    foundTarget = false;
+                    break;
+                } else {
+                    childRef = childInputs.get(0);
+                    child = (AbstractLogicalOperator) childRef.getValue();
+                }
+            }
+            /** the target order-by operator has not been found. */
+            if (!foundTarget) {
+                return false;
+            }
+
+            /** copy the original order-by operator and insert on-top-of the subplan operator */
+            context.addToDontApplySet(this, child);
+            OrderOperator sourceOrderOp = (OrderOperator) child;
+            for (Pair<IOrder, Mutable<ILogicalExpression>> expr : sourceOrderOp.getOrderExpressions()) {
+                if (!expr.second.getValue().isFunctional()) {
+                    return false;
+                }
+            }
+            List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExprs = deepCopyOrderAndExpression(sourceOrderOp
+                    .getOrderExpressions());
+            OrderOperator newOrderOp = new OrderOperator(orderExprs);
+            context.addToDontApplySet(this, newOrderOp);
+            inputs.set(i, new MutableObject<ILogicalOperator>(newOrderOp));
+            newOrderOp.getInputs().add(inputOpRef);
+            context.computeAndSetTypeEnvironmentForOperator(newOrderOp);
+
+            if (!orderSensitive) {
+                /** remove the original order-by */
+                childRef.setValue(sourceOrderOp.getInputs().get(0).getValue());
+            }
+            changed = true;
+        }
+        return changed;
+    }
+
+    private Mutable<ILogicalExpression> deepCopyExpressionRef(Mutable<ILogicalExpression> oldExpr) {
+        return new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression());
+    }
+
+    private List<Pair<IOrder, Mutable<ILogicalExpression>>> deepCopyOrderAndExpression(
+            List<Pair<IOrder, Mutable<ILogicalExpression>>> ordersAndExprs) {
+        List<Pair<IOrder, Mutable<ILogicalExpression>>> newOrdersAndExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+        for (Pair<IOrder, Mutable<ILogicalExpression>> pair : ordersAndExprs)
+            newOrdersAndExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(pair.first,
+                    deepCopyExpressionRef(pair.second)));
+        return newOrdersAndExprs;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
new file mode 100644
index 0000000..ebd7da1
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
@@ -0,0 +1,614 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.BroadcastPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreSortedDistinctByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RandomMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RandomPartitionPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionMergePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.DefaultNodeGroupDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty.PartitioningType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.range.IRangeMap;
+
+public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule {
+
+    private static final INodeDomain DEFAULT_DOMAIN = new DefaultNodeGroupDomain("__DEFAULT");
+
+    private PhysicalOptimizationConfig physicalOptimizationConfig;
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        // wait for the physical operators to be set first
+        if (op.getPhysicalOperator() == null) {
+            return false;
+        }
+        if (context.checkIfInDontApplySet(this, op)) {
+            return false;
+        }
+
+        List<FunctionalDependency> fds = context.getFDList(op);
+        if (fds != null && !fds.isEmpty()) {
+            return false;
+        }
+        // These are actually logical constraints, so they could be pre-computed
+        // somewhere else, too.
+
+        physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Optimizing operator " + op.getPhysicalOperator() + ".\n");
+
+        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
+
+        StructuralPropertiesVector pvector = new StructuralPropertiesVector(new RandomPartitioningProperty(null),
+                new LinkedList<ILocalStructuralProperty>());
+        boolean changed = physOptimizeOp(opRef, pvector, false, context);
+        op.computeDeliveredPhysicalProperties(context);
+        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator() + ": "
+                + op.getDeliveredPhysicalProperties() + "\n");
+
+        context.addToDontApplySet(this, opRef.getValue());
+
+        return changed;
+    }
+
+    private boolean physOptimizePlan(ILogicalPlan plan, IPhysicalPropertiesVector pvector, boolean nestedPlan,
+            IOptimizationContext context) throws AlgebricksException {
+        boolean changed = false;
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            if (physOptimizeOp(root, pvector, nestedPlan, context)) {
+                changed = true;
+            }
+            AbstractLogicalOperator op = (AbstractLogicalOperator) root.getValue();
+            op.computeDeliveredPhysicalProperties(context);
+            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator()
+                    + ": " + op.getDeliveredPhysicalProperties() + "\n");
+        }
+        return changed;
+    }
+
+    private boolean physOptimizeOp(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector required,
+            boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
+
+        boolean changed = false;
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        optimizeUsingConstraintsAndEquivClasses(op);
+        PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required);
+        IPhysicalPropertiesVector[] reqdProperties = null;
+        if (pr != null) {
+            reqdProperties = pr.getRequiredProperties();
+        }
+        boolean opIsRedundantSort = false;
+
+        // compute properties and figure out the domain
+        INodeDomain childrenDomain = null;
+        {
+            int j = 0;
+            for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+                AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
+                // recursive call
+                if (physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context)) {
+                    changed = true;
+                }
+                child.computeDeliveredPhysicalProperties(context);
+                IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
+                if (childrenDomain == null) {
+                    childrenDomain = delivered.getPartitioningProperty().getNodeDomain();
+                } else {
+                    INodeDomain dom2 = delivered.getPartitioningProperty().getNodeDomain();
+                    if (!childrenDomain.sameAs(dom2)) {
+                        childrenDomain = DEFAULT_DOMAIN;
+                    }
+                }
+                j++;
+            }
+        }
+
+        if (reqdProperties != null) {
+            for (int k = 0; k < reqdProperties.length; k++) {
+                IPhysicalPropertiesVector pv = reqdProperties[k];
+                IPartitioningProperty pp = pv.getPartitioningProperty();
+                if (pp != null && pp.getNodeDomain() == null) {
+                    pp.setNodeDomain(childrenDomain);
+                }
+            }
+        }
+
+        IPartitioningProperty firstDeliveredPartitioning = null;
+        int i = 0;
+        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
+            IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
+
+            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Properties delivered by " + child.getPhysicalOperator()
+                    + ": " + delivered + "\n");
+            IPartitioningRequirementsCoordinator prc = pr.getPartitioningCoordinator();
+            Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(
+                    reqdProperties[i].getPartitioningProperty(), firstDeliveredPartitioning, op, context);
+            boolean mayExpandPartitioningProperties = pbpp.first;
+            IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second,
+                    reqdProperties[i].getLocalProperties());
+
+            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for " + child.getPhysicalOperator()
+                    + ": " + rqd + "\n");
+            IPhysicalPropertiesVector diff = delivered.getUnsatisfiedPropertiesFrom(rqd,
+                    mayExpandPartitioningProperties, context.getEquivalenceClassMap(child), context.getFDList(child));
+
+            if (isRedundantSort(opRef, delivered, diff, context)) {
+                opIsRedundantSort = true;
+            }
+
+            if (diff != null) {
+                changed = true;
+                addEnforcers(op, i, diff, rqd, delivered, childrenDomain, nestedPlan, context);
+
+                AbstractLogicalOperator newChild = ((AbstractLogicalOperator) op.getInputs().get(i).getValue());
+
+                if (newChild != child) {
+                    delivered = newChild.getDeliveredPhysicalProperties();
+                    IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, rqd,
+                            mayExpandPartitioningProperties, context);
+                    AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
+
+                    if (isRedundantSort(opRef, delivered, newDiff, context)) {
+                        opIsRedundantSort = true;
+                        break;
+                    }
+                }
+
+            }
+            if (firstDeliveredPartitioning == null) {
+                IPartitioningProperty dpp = delivered.getPartitioningProperty();
+                if (dpp.getPartitioningType() == PartitioningType.ORDERED_PARTITIONED
+                        || dpp.getPartitioningType() == PartitioningType.UNORDERED_PARTITIONED) {
+                    firstDeliveredPartitioning = dpp;
+                }
+            }
+
+            i++;
+        }
+
+        if (op.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
+            for (ILogicalPlan p : nested.getNestedPlans()) {
+                if (physOptimizePlan(p, required, true, context)) {
+                    changed = true;
+                }
+            }
+        }
+
+        if (opIsRedundantSort) {
+            if (AlgebricksConfig.DEBUG) {
+                AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Removing redundant SORT operator "
+                        + op.getPhysicalOperator() + "\n");
+                printOp(op);
+            }
+            changed = true;
+            AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+            if (nextOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+                nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getValue();
+            }
+            opRef.setValue(nextOp);
+            // Now, transfer annotations from the original sort op. to this one.
+            AbstractLogicalOperator transferTo = nextOp;
+            if (transferTo.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+                //
+                // remove duplicate exchange operator
+                transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getValue();
+            }
+            transferTo.getAnnotations().putAll(op.getAnnotations());
+            physOptimizeOp(opRef, required, nestedPlan, context);
+        }
+        return changed;
+    }
+
+    private IPhysicalPropertiesVector newPropertiesDiff(AbstractLogicalOperator newChild,
+            IPhysicalPropertiesVector required, boolean mayExpandPartitioningProperties, IOptimizationContext context)
+            throws AlgebricksException {
+        IPhysicalPropertiesVector newDelivered = newChild.getDeliveredPhysicalProperties();
+
+        Map<LogicalVariable, EquivalenceClass> newChildEqClasses = context.getEquivalenceClassMap(newChild);
+        List<FunctionalDependency> newChildFDs = context.getFDList(newChild);
+        if (newChildEqClasses == null || newChildFDs == null) {
+            FDsAndEquivClassesVisitor fdsVisitor = new FDsAndEquivClassesVisitor();
+            newChild.accept(fdsVisitor, context);
+            newChildEqClasses = context.getEquivalenceClassMap(newChild);
+            newChildFDs = context.getFDList(newChild);
+        }
+        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for new op. "
+                + newChild.getPhysicalOperator() + ": " + required + "\n");
+
+        return newDelivered.getUnsatisfiedPropertiesFrom(required, mayExpandPartitioningProperties, newChildEqClasses,
+                newChildFDs);
+    }
+
+    private void optimizeUsingConstraintsAndEquivClasses(AbstractLogicalOperator op) {
+        IPhysicalOperator pOp = op.getPhysicalOperator();
+        switch (pOp.getOperatorTag()) {
+            case HASH_GROUP_BY:
+            case EXTERNAL_GROUP_BY: {
+                GroupByOperator gby = (GroupByOperator) op;
+                ExternalGroupByPOperator hgbyOp = (ExternalGroupByPOperator) pOp;
+                hgbyOp.computeColumnSet(gby.getGroupByList());
+                break;
+            }
+            case PRE_CLUSTERED_GROUP_BY: {
+                GroupByOperator gby = (GroupByOperator) op;
+                PreclusteredGroupByPOperator preSortedGby = (PreclusteredGroupByPOperator) pOp;
+                preSortedGby.setGbyColumns(gby.getGbyVarList());
+                break;
+            }
+            case PRE_SORTED_DISTINCT_BY: {
+                DistinctOperator d = (DistinctOperator) op;
+                PreSortedDistinctByPOperator preSortedDistinct = (PreSortedDistinctByPOperator) pOp;
+                preSortedDistinct.setDistinctByColumns(d.getDistinctByVarList());
+                break;
+            }
+        }
+    }
+
+    private List<OrderColumn> getOrderColumnsFromGroupingProperties(List<ILocalStructuralProperty> reqd,
+            List<ILocalStructuralProperty> dlvd) {
+        List<OrderColumn> returnedProperties = new ArrayList<OrderColumn>();
+        List<LogicalVariable> rqdCols = new ArrayList<LogicalVariable>();
+        List<LogicalVariable> dlvdCols = new ArrayList<LogicalVariable>();
+        for (ILocalStructuralProperty r : reqd) {
+            r.getVariables(rqdCols);
+        }
+        for (ILocalStructuralProperty d : dlvd) {
+            d.getVariables(dlvdCols);
+        }
+
+        int prefix = dlvdCols.size() - 1;
+        for (; prefix >= 0;) {
+            if (!rqdCols.contains(dlvdCols.get(prefix))) {
+                prefix--;
+            } else {
+                break;
+            }
+        }
+
+        LocalOrderProperty orderProp = (LocalOrderProperty) dlvd.get(0);
+        List<OrderColumn> orderColumns = orderProp.getOrderColumns();
+        for (int j = 0; j <= prefix; j++) {
+            returnedProperties.add(new OrderColumn(orderColumns.get(j).getColumn(), orderColumns.get(j).getOrder()));
+        }
+        // maintain other order columns after the required order columns
+        if (returnedProperties.size() != 0) {
+            for (int j = prefix + 1; j < dlvdCols.size(); j++) {
+                OrderColumn oc = orderColumns.get(j);
+                returnedProperties.add(new OrderColumn(oc.getColumn(), oc.getOrder()));
+            }
+        }
+        return returnedProperties;
+    }
+
+    /*
+     * We assume delivered to be already normalized.
+     */
+    private boolean isRedundantSort(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector delivered,
+            IPhysicalPropertiesVector diffOfProperties, IOptimizationContext context) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.ORDER
+                || (op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && op
+                        .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT)
+                || delivered.getLocalProperties() == null) {
+            return false;
+        }
+        AbstractStableSortPOperator sortOp = (AbstractStableSortPOperator) op.getPhysicalOperator();
+        sortOp.computeLocalProperties(op);
+        ILocalStructuralProperty orderProp = sortOp.getOrderProperty();
+        return PropertiesUtil.matchLocalProperties(Collections.singletonList(orderProp),
+                delivered.getLocalProperties(), context.getEquivalenceClassMap(op), context.getFDList(op));
+    }
+
+    private void addEnforcers(AbstractLogicalOperator op, int childIndex,
+            IPhysicalPropertiesVector diffPropertiesVector, IPhysicalPropertiesVector required,
+            IPhysicalPropertiesVector deliveredByChild, INodeDomain domain, boolean nestedPlan,
+            IOptimizationContext context) throws AlgebricksException {
+
+        IPartitioningProperty pp = diffPropertiesVector.getPartitioningProperty();
+        if (pp == null || pp.getPartitioningType() == PartitioningType.UNPARTITIONED) {
+            addLocalEnforcers(op, childIndex, diffPropertiesVector.getLocalProperties(), nestedPlan, context);
+            IPhysicalPropertiesVector deliveredByNewChild = ((AbstractLogicalOperator) op.getInputs().get(0).getValue())
+                    .getDeliveredPhysicalProperties();
+            addPartitioningEnforcers(op, childIndex, pp, required, deliveredByNewChild, domain, context);
+        } else {
+            addPartitioningEnforcers(op, childIndex, pp, required, deliveredByChild, domain, context);
+            AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
+            IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, required, true, context);
+            AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
+            if (newDiff != null) {
+                addLocalEnforcers(op, childIndex, newDiff.getLocalProperties(), nestedPlan, context);
+            }
+        }
+    }
+
+    private void addLocalEnforcers(AbstractLogicalOperator op, int i, List<ILocalStructuralProperty> localProperties,
+            boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
+        if (AlgebricksConfig.DEBUG) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Adding local enforcers for local props = " + localProperties
+                    + "\n");
+        }
+
+        if (localProperties == null || localProperties.isEmpty()) {
+            return;
+        }
+
+        Mutable<ILogicalOperator> topOp = new MutableObject<ILogicalOperator>();
+        topOp.setValue(op.getInputs().get(i).getValue());
+        LinkedList<LocalOrderProperty> oList = new LinkedList<LocalOrderProperty>();
+
+        for (ILocalStructuralProperty prop : localProperties) {
+            switch (prop.getPropertyType()) {
+                case LOCAL_ORDER_PROPERTY: {
+                    oList.add((LocalOrderProperty) prop);
+                    break;
+                }
+                case LOCAL_GROUPING_PROPERTY: {
+                    LocalGroupingProperty g = (LocalGroupingProperty) prop;
+                    Collection<LogicalVariable> vars = (g.getPreferredOrderEnforcer() != null) ? g
+                            .getPreferredOrderEnforcer() : g.getColumnSet();
+                    List<OrderColumn> orderColumns = new ArrayList<OrderColumn>();
+                    for (LogicalVariable v : vars) {
+                        OrderColumn oc = new OrderColumn(v, OrderKind.ASC);
+                        orderColumns.add(oc);
+                    }
+                    LocalOrderProperty lop = new LocalOrderProperty(orderColumns);
+                    oList.add(lop);
+                    break;
+                }
+                default: {
+                    throw new IllegalStateException();
+                }
+            }
+        }
+        if (!oList.isEmpty()) {
+            topOp = enforceOrderProperties(oList, topOp, nestedPlan, context);
+        }
+
+        op.getInputs().set(i, topOp);
+        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) topOp.getValue(), context);
+        printOp((AbstractLogicalOperator) topOp.getValue());
+    }
+
+    private Mutable<ILogicalOperator> enforceOrderProperties(List<LocalOrderProperty> oList,
+            Mutable<ILogicalOperator> topOp, boolean isMicroOp, IOptimizationContext context)
+            throws AlgebricksException {
+        List<Pair<IOrder, Mutable<ILogicalExpression>>> oe = new LinkedList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+        for (LocalOrderProperty orderProperty : oList) {
+            for (OrderColumn oc : orderProperty.getOrderColumns()) {
+                IOrder ordType = (oc.getOrder() == OrderKind.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
+                Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<IOrder, Mutable<ILogicalExpression>>(ordType,
+                        new MutableObject<ILogicalExpression>(new VariableReferenceExpression(oc.getColumn())));
+                oe.add(pair);
+            }
+        }
+        OrderOperator oo = new OrderOperator(oe);
+        oo.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+        if (isMicroOp) {
+            oo.setPhysicalOperator(new InMemoryStableSortPOperator());
+        } else {
+            oo.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort()));
+        }
+        oo.getInputs().add(topOp);
+        context.computeAndSetTypeEnvironmentForOperator(oo);
+        if (AlgebricksConfig.DEBUG) {
+            AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added sort enforcer " + oo.getPhysicalOperator() + ".\n");
+        }
+        return new MutableObject<ILogicalOperator>(oo);
+    }
+
+    private void addPartitioningEnforcers(ILogicalOperator op, int i, IPartitioningProperty pp,
+            IPhysicalPropertiesVector required, IPhysicalPropertiesVector deliveredByChild, INodeDomain domain,
+            IOptimizationContext context) throws AlgebricksException {
+        if (pp != null) {
+            IPhysicalOperator pop;
+            switch (pp.getPartitioningType()) {
+                case UNPARTITIONED: {
+                    List<OrderColumn> ordCols = computeOrderColumns(deliveredByChild);
+                    if (ordCols == null || ordCols.size() == 0) {
+                        pop = new RandomMergeExchangePOperator();
+                    } else {
+                        if (op.getAnnotations().containsKey(OperatorAnnotations.USE_RANGE_CONNECTOR)) {
+                            IRangeMap rangeMap = (IRangeMap) op.getAnnotations().get(
+                                    OperatorAnnotations.USE_RANGE_CONNECTOR);
+                            pop = new RangePartitionMergePOperator(ordCols, domain, rangeMap);
+                        } else {
+                            OrderColumn[] sortColumns = new OrderColumn[ordCols.size()];
+                            sortColumns = ordCols.toArray(sortColumns);
+                            pop = new SortMergeExchangePOperator(sortColumns);
+                        }
+                    }
+                    break;
+                }
+                case UNORDERED_PARTITIONED: {
+                    List<LogicalVariable> varList = new ArrayList<LogicalVariable>(
+                            ((UnorderedPartitionedProperty) pp).getColumnSet());
+                    List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties();
+                    List<ILocalStructuralProperty> reqdLocals = required.getLocalProperties();
+                    boolean propWasSet = false;
+                    pop = null;
+                    if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) {
+                        AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
+                        Map<LogicalVariable, EquivalenceClass> ecs = context.getEquivalenceClassMap(c);
+                        List<FunctionalDependency> fds = context.getFDList(c);
+                        if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) {
+                            List<OrderColumn> orderColumns = getOrderColumnsFromGroupingProperties(reqdLocals,
+                                    cldLocals);
+                            pop = new HashPartitionMergeExchangePOperator(orderColumns, varList, domain);
+                            propWasSet = true;
+                        }
+                    }
+                    if (!propWasSet) {
+                        pop = new HashPartitionExchangePOperator(varList, domain);
+                    }
+                    break;
+                }
+                case ORDERED_PARTITIONED: {
+                    pop = new RangePartitionPOperator(((OrderedPartitionedProperty) pp).getOrderColumns(), domain, null);
+                    break;
+                }
+                case BROADCAST: {
+                    pop = new BroadcastPOperator(domain);
+                    break;
+                }
+                case RANDOM: {
+                    RandomPartitioningProperty rpp = (RandomPartitioningProperty) pp;
+                    INodeDomain nd = rpp.getNodeDomain();
+                    pop = new RandomPartitionPOperator(nd);
+                    break;
+                }
+                default: {
+                    throw new NotImplementedException("Enforcer for " + pp.getPartitioningType()
+                            + " partitioning type has not been implemented.");
+                }
+            }
+            Mutable<ILogicalOperator> ci = op.getInputs().get(i);
+            ExchangeOperator exchg = new ExchangeOperator();
+            exchg.setPhysicalOperator(pop);
+            setNewOp(ci, exchg, context);
+            exchg.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+            OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(exchg, context);
+            context.computeAndSetTypeEnvironmentForOperator(exchg);
+            if (AlgebricksConfig.DEBUG) {
+                AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added partitioning enforcer "
+                        + exchg.getPhysicalOperator() + ".\n");
+                printOp((AbstractLogicalOperator) op);
+            }
+        }
+    }
+
+    private boolean allAreOrderProps(List<ILocalStructuralProperty> cldLocals) {
+        for (ILocalStructuralProperty lsp : cldLocals) {
+            if (lsp.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+                return false;
+            }
+        }
+        return !cldLocals.isEmpty();
+    }
+
+    private void printOp(AbstractLogicalOperator op) throws AlgebricksException {
+        StringBuilder sb = new StringBuilder();
+        LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
+        PlanPrettyPrinter.printOperator(op, sb, pvisitor, 0);
+        AlgebricksConfig.ALGEBRICKS_LOGGER.fine(sb.toString());
+    }
+
+    private List<OrderColumn> computeOrderColumns(IPhysicalPropertiesVector pv) {
+        List<OrderColumn> ordCols = new ArrayList<OrderColumn>();
+        List<ILocalStructuralProperty> localProps = pv.getLocalProperties();
+        if (localProps == null || localProps.size() == 0) {
+            return null;
+        } else {
+            for (ILocalStructuralProperty p : localProps) {
+                if (p.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+                    LocalOrderProperty lop = (LocalOrderProperty) p;
+                    ordCols.addAll(lop.getOrderColumns());
+                } else {
+                    return null;
+                }
+            }
+            return ordCols;
+        }
+
+    }
+
+    private void setNewOp(Mutable<ILogicalOperator> opRef, AbstractLogicalOperator newOp, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator oldOp = opRef.getValue();
+        opRef.setValue(newOp);
+        newOp.getInputs().add(new MutableObject<ILogicalOperator>(oldOp));
+        newOp.recomputeSchema();
+        newOp.computeDeliveredPhysicalProperties(context);
+        context.computeAndSetTypeEnvironmentForOperator(newOp);
+        AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + newOp.getPhysicalOperator()
+                + ": " + newOp.getDeliveredPhysicalProperties() + "\n");
+
+        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(newOp, context);
+    }
+
+}

[24/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
new file mode 100644
index 0000000..2c4d89d
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+
+/**
+ * This will always be attached to an {@link OrderOperator} logical operator.
+ */
+
+public class StableSortPOperator extends AbstractStableSortPOperator {
+
+    private int maxNumberOfFrames;
+
+    public StableSortPOperator(int maxNumberOfFrames) {
+        super();
+        this.maxNumberOfFrames = maxNumberOfFrames;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.STABLE_SORT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        int n = sortColumns.length;
+        int[] sortFields = new int[n];
+        IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+
+        INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+        INormalizedKeyComputerFactory nkcf = null;
+
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        int i = 0;
+        for (OrderColumn oc : sortColumns) {
+            LogicalVariable var = oc.getColumn();
+            sortFields[i] = opSchema.findVariable(var);
+            Object type = env.getVarType(var);
+            OrderKind order = oc.getOrder();
+            if (i == 0 && nkcfProvider != null && type != null) {
+                nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order == OrderKind.ASC);
+            }
+            IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+            comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder() == OrderKind.ASC);
+            i++;
+        }
+
+        ExternalSortOperatorDescriptor sortOpDesc = new ExternalSortOperatorDescriptor(spec, maxNumberOfFrames,
+                sortFields, nkcf, comps, recDescriptor);
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
new file mode 100644
index 0000000..63ddffe
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamLimitRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamLimitPOperator extends AbstractPhysicalOperator {
+
+    public StreamLimitPOperator() {
+
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.STREAM_LIMIT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator limitOp = (AbstractLogicalOperator) op;
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        if (limitOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+            //partitioning property: unpartitioned;  local property: whatever from the child
+            deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, op2
+                    .getDeliveredPhysicalProperties().getLocalProperties());
+        } else {
+            deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+        }
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        AbstractLogicalOperator limitOp = (AbstractLogicalOperator) op;
+        if (limitOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+            StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+            pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
+            return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+        } else {
+            return emptyUnaryRequirements();
+        }
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        LimitOperator limit = (LimitOperator) op;
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+        IScalarEvaluatorFactory maxObjectsFact = expressionRuntimeProvider.createEvaluatorFactory(limit.getMaxObjects()
+                .getValue(), env, inputSchemas, context);
+        ILogicalExpression offsetExpr = limit.getOffset().getValue();
+        IScalarEvaluatorFactory offsetFact = (offsetExpr == null) ? null : expressionRuntimeProvider
+                .createEvaluatorFactory(offsetExpr, env, inputSchemas, context);
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
+                context);
+        StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null,
+                context.getBinaryIntegerInspectorFactory());
+        builder.contributeMicroOperator(limit, runtime, recDesc);
+        // and contribute one edge from its child
+        ILogicalOperator src = limit.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, limit, 0);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
new file mode 100644
index 0000000..2ad5fda
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamProjectRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamProjectPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
+
+    private boolean flushFramesRapidly;
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.STREAM_PROJECT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        ProjectOperator project = (ProjectOperator) op;
+        int[] projectionList = new int[project.getVariables().size()];
+        int i = 0;
+        for (LogicalVariable v : project.getVariables()) {
+            int pos = inputSchemas[0].findVariable(v);
+            if (pos < 0) {
+                throw new AlgebricksException("Could not find variable " + v + ".");
+            }
+            projectionList[i++] = pos;
+        }
+        StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList, flushFramesRapidly);
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
+                context);
+        builder.contributeMicroOperator(project, runtime, recDesc);
+        ILogicalOperator src = project.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, project, 0);
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        ProjectOperator p = (ProjectOperator) op;
+        computeDeliveredPropertiesForUsedVariables(p, p.getVariables());
+    }
+
+    public void setRapidFrameFlush(boolean flushFramesRapidly) {
+        this.flushFramesRapidly = flushFramesRapidly;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
new file mode 100644
index 0000000..3999fa6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StreamSelectRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamSelectPOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.STREAM_SELECT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        SelectOperator select = (SelectOperator) op;
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(select.getCondition()
+                .getValue(), context.getTypeEnvironment(op), inputSchemas, context);
+        StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null,
+                context.getBinaryBooleanInspectorFactory(), select.getRetainNull(), inputSchemas[0].findVariable(select
+                        .getNullPlaceholderVariable()), context.getNullWriterFactory());
+        // contribute one Asterix framewriter
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        builder.contributeMicroOperator(select, runtime, recDesc);
+        // and contribute one edge from its child
+        ILogicalOperator src = select.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, select, 0);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
new file mode 100644
index 0000000..e6a0be7
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription.ScriptKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.StringStreamingScriptDescription;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.StringStreamingRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StringStreamingScriptPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.STRING_STREAM_SCRIPT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        ScriptOperator scriptOp = (ScriptOperator) op;
+        IScriptDescription scriptDesc = scriptOp.getScriptDescription();
+        if (scriptDesc.getKind() != ScriptKind.STRING_STREAMING) {
+            throw new IllegalStateException();
+        }
+        StringStreamingScriptDescription sssd = (StringStreamingScriptDescription) scriptDesc;
+        StringStreamingRuntimeFactory runtime = new StringStreamingRuntimeFactory(sssd.getCommand(),
+                sssd.getPrinterFactories(), sssd.getFieldDelimiter(), sssd.getParserFactory());
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema,
+                context);
+        builder.contributeMicroOperator(scriptOp, runtime, recDesc);
+        // and contribute one edge from its child
+        ILogicalOperator src = scriptOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, scriptOp, 0);
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        ScriptOperator s = (ScriptOperator) op;
+        computeDeliveredPropertiesForUsedVariables(s, s.getInputVariables());
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
new file mode 100644
index 0000000..efc609a
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.meta.SubplanRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class SubplanPOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.SUBPLAN;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        IPhysicalPropertiesVector childsProperties = op2.getPhysicalOperator().getDeliveredProperties();
+        List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
+        if (childsProperties.getLocalProperties() != null) {
+            propsLocal.addAll(childsProperties.getLocalProperties());
+        }
+        // ... get local properties for newly created variables...
+        SubplanOperator subplan = (SubplanOperator) op;
+        for (ILogicalPlan plan : subplan.getNestedPlans()) {
+            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+                AbstractLogicalOperator rOp = (AbstractLogicalOperator) r.getValue();
+                propsLocal.addAll(rOp.getPhysicalOperator().getDeliveredProperties().getLocalProperties());
+            }
+        }
+
+        deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal);
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        SubplanOperator subplan = (SubplanOperator) op;
+        if (subplan.getNestedPlans().size() != 1) {
+            throw new NotImplementedException("Subplan currently works only for one nested plan with one root.");
+        }
+        AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], subplan, opSchema, context);
+        assert (subplans.length == 1);
+        AlgebricksPipeline np = subplans[0];
+        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op.getInputs().get(0).getValue()),
+                inputSchemas[0], context);
+        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[np.getOutputWidth()];
+        for (int i = 0; i < nullWriterFactories.length; i++) {
+            nullWriterFactories[i] = context.getNullWriterFactory();
+        }
+        SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, nullWriterFactories, inputRecordDesc, null);
+
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        builder.contributeMicroOperator(subplan, runtime, recDesc);
+
+        ILogicalOperator src = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, op, 0);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java
new file mode 100644
index 0000000..8a2ecf6
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/TokenizePOperator.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class TokenizePOperator extends AbstractPhysicalOperator {
+
+    private final List<LogicalVariable> primaryKeys;
+    private final List<LogicalVariable> secondaryKeys;
+    private final IDataSourceIndex<?, ?> dataSourceIndex;
+
+    public TokenizePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            IDataSourceIndex<?, ?> dataSourceIndex) {
+        this.primaryKeys = primaryKeys;
+        this.secondaryKeys = secondaryKeys;
+        this.dataSourceIndex = dataSourceIndex;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.TOKENIZE;
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        return emptyUnaryRequirements();
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        TokenizeOperator tokenizeOp = (TokenizeOperator) op;
+        if (tokenizeOp.getOperation() != Kind.INSERT || !tokenizeOp.isBulkload()) {
+            throw new AlgebricksException("Tokenize Operator only works when bulk-loading data.");
+        }
+
+        IMetadataProvider mp = context.getMetadataProvider();
+        IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
+        JobSpecification spec = builder.getJobSpec();
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context);
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getTokenizerRuntime(
+                dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, null, inputDesc,
+                context, spec, true);
+        builder.contributeHyracksOperator(tokenizeOp, runtimeAndConstraints.first);
+        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+        ILogicalOperator src = tokenizeOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, tokenizeOp, 0);
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
new file mode 100644
index 0000000..cda3b00
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.IOperatorDescriptorRegistry;
+import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
+
+public class UnionAllPOperator extends AbstractPhysicalOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.UNION_ALL;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return false;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
+        this.deliveredProperties = new StructuralPropertiesVector(pp, new ArrayList<ILocalStructuralProperty>(0));
+
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        StructuralPropertiesVector pv0 = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
+        StructuralPropertiesVector pv1 = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
+        return new PhysicalRequirements(new StructuralPropertiesVector[] { pv0, pv1 },
+                IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        UnionAllOperator unionOp = (UnionAllOperator) op;
+        int n = unionOp.getVariableMappings().size();
+        int[] leftColumns = new int[n];
+        int[] rightColumns = new int[n];
+        int i = 0;
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : unionOp.getVariableMappings()) {
+            int posLeft = inputSchemas[0].findVariable(t.first);
+            leftColumns[i] = posLeft;
+            int posRight = inputSchemas[1].findVariable(t.second);
+            rightColumns[i] = posRight;
+            ++i;
+        }
+
+        IOperatorDescriptorRegistry spec = builder.getJobSpec();
+        RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+
+        // at algebricks level, union all only accepts two inputs, although at
+        // hyracks
+        // level, there is no restrictions
+        UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(spec, 2, recordDescriptor);
+        contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src1, 0, op, 0);
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
+        builder.contributeGraphEdge(src2, 0, op, 1);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
new file mode 100644
index 0000000..d60d4f7
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionRuntimeProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class UnnestPOperator extends AbstractScanPOperator {
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.UNNEST;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        UnnestOperator unnest = (UnnestOperator) op;
+
+        int outCol = opSchema.findVariable(unnest.getVariable());
+        ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
+        IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
+        boolean exit = false;
+        if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            exit = true;
+        } else {
+            AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) unnestExpr;
+            if (fce.getKind() != FunctionKind.UNNEST) {
+                exit = true;
+            }
+        }
+        if (exit) {
+            throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call.");
+        }
+        UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr;
+        IUnnestingEvaluatorFactory unnestingFactory = expressionRuntimeProvider.createUnnestingFunctionFactory(agg,
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+
+        // for position offset
+        ILogicalExpression posOffsetExpr = unnest.getPositionOffsetExpr();
+        IScalarEvaluatorFactory posOffsetExprEvalFactory = null;
+        if (posOffsetExpr != null) {
+            posOffsetExprEvalFactory = expressionRuntimeProvider.createEvaluatorFactory(posOffsetExpr,
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+        }
+
+        int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
+        UnnestRuntimeFactory unnestRuntime = new UnnestRuntimeFactory(outCol, unnestingFactory, projectionList,
+                unnest.getPositionWriter(), posOffsetExprEvalFactory);
+        RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context);
+        builder.contributeMicroOperator(unnest, unnestRuntime, recDesc);
+        ILogicalOperator src = unnest.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, unnest, 0);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
new file mode 100644
index 0000000..f399184
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class WriteResultPOperator extends AbstractPhysicalOperator {
+
+    private LogicalVariable payload;
+    private List<LogicalVariable> keys;
+    private IDataSource<?> dataSource;
+    private final List<LogicalVariable> additionalFilteringKeys;
+
+    public WriteResultPOperator(IDataSource<?> dataSource, LogicalVariable payload, List<LogicalVariable> keys,
+            List<LogicalVariable> additionalFilteringKeys) {
+        this.dataSource = dataSource;
+        this.payload = payload;
+        this.keys = keys;
+        this.additionalFilteringKeys = additionalFilteringKeys;
+    }
+
+    @Override
+    public String toString() {
+        return getOperatorTag().toString() + " " + payload + " " + keys;
+    }
+
+    @Override
+    public PhysicalOperatorTag getOperatorTag() {
+        return PhysicalOperatorTag.WRITE_RESULT;
+    }
+
+    @Override
+    public boolean isMicroOperator() {
+        return true;
+    }
+
+    @Override
+    public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+    }
+
+    @Override
+    public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+            IPhysicalPropertiesVector reqdByParent) {
+        List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
+        scanVariables.addAll(keys);
+        scanVariables.add(new LogicalVariable(-1));
+        IPhysicalPropertiesVector r = dataSource.getPropertiesProvider().computePropertiesVector(scanVariables);
+        IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
+        requirements[0] = r;
+        return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+    }
+
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    @Override
+    public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+            throws AlgebricksException {
+        WriteResultOperator writeResultOp = (WriteResultOperator) op;
+        IMetadataProvider mp = context.getMetadataProvider();
+
+        JobSpecification spec = builder.getJobSpec();
+        Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getWriteResultRuntime(
+                dataSource, propagatedSchema, keys, payload, additionalFilteringKeys, context, spec);
+
+        builder.contributeHyracksOperator(writeResultOp, runtimeAndConstraints.first);
+        builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+        ILogicalOperator src = writeResultOp.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(src, 0, writeResultOp, 0);
+    }
+
+    @Override
+    public boolean expensiveThanMaterialization() {
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
new file mode 100644
index 0000000..8048bf1
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.plan;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+
+/*
+ * Author: Guangqiang Li
+ * Created on Jul 9, 2009 
+ */
+public class ALogicalPlanImpl implements ILogicalPlan {
+    private List<Mutable<ILogicalOperator>> roots;
+
+    public ALogicalPlanImpl() {
+        this.roots = new ArrayList<Mutable<ILogicalOperator>>();
+    }
+
+    public ALogicalPlanImpl(List<Mutable<ILogicalOperator>> roots) {
+        this.roots = roots;
+    }
+
+    public ALogicalPlanImpl(Mutable<ILogicalOperator> root) {
+        roots = new ArrayList<Mutable<ILogicalOperator>>(1);
+        roots.add(root);
+    }
+
+    public List<Mutable<ILogicalOperator>> getRoots() {
+        return roots;
+    }
+
+    public void setRoots(List<Mutable<ILogicalOperator>> roots) {
+        this.roots = roots;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
new file mode 100644
index 0000000..5f77352
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalExpressionPrettyPrintVisitor.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2014 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+
+
+public class LogicalExpressionPrettyPrintVisitor implements ILogicalExpressionVisitor<String, Integer> {
+
+	@Override
+	public String visitConstantExpression(ConstantExpression expr, Integer indent)
+			throws AlgebricksException {
+		return expr.toString();
+	}
+
+	@Override
+	public String visitVariableReferenceExpression(
+			VariableReferenceExpression expr, Integer indent)
+			throws AlgebricksException {
+		return expr.toString();
+	}
+
+	@Override
+	public String visitAggregateFunctionCallExpression(
+			AggregateFunctionCallExpression expr, Integer indent)
+			throws AlgebricksException {
+		return expr.toString();
+	}
+
+	@Override
+	public String visitScalarFunctionCallExpression(
+			ScalarFunctionCallExpression expr, Integer indent)
+			throws AlgebricksException {
+		return expr.toString();
+	}
+
+	@Override
+	public String visitStatefulFunctionCallExpression(
+			StatefulFunctionCallExpression expr, Integer indent)
+			throws AlgebricksException {
+		return expr.toString();
+	}
+
+	@Override
+	public String visitUnnestingFunctionCallExpression(
+			UnnestingFunctionCallExpression expr, Integer indent)
+			throws AlgebricksException {
+		return expr.toString();
+	}
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
new file mode 100644
index 0000000..2c924d5
--- /dev/null
+++ b/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -0,0 +1,431 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+
+public class LogicalOperatorPrettyPrintVisitor implements ILogicalOperatorVisitor<String, Integer> {
+
+    ILogicalExpressionVisitor<String, Integer> exprVisitor;
+
+    public LogicalOperatorPrettyPrintVisitor() {
+        exprVisitor = new LogicalExpressionPrettyPrintVisitor();
+    }
+
+    public LogicalOperatorPrettyPrintVisitor(ILogicalExpressionVisitor<String, Integer> exprVisitor) {
+        this.exprVisitor = exprVisitor;
+    }
+
+    @Override
+    public String visitAggregateOperator(AggregateOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("aggregate ").append(op.getVariables()).append(" <- ");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitRunningAggregateOperator(RunningAggregateOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("running-aggregate ").append(op.getVariables()).append(" <- ");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("empty-tuple-source");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("group by (");
+        pprintVeList(buffer, op.getGroupByList(), indent);
+        buffer.append(") decor (");
+        pprintVeList(buffer, op.getDecorList(), indent);
+        buffer.append(") {");
+        printNestedPlans(op, indent, buffer);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitDistinctOperator(DistinctOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("distinct " + "(");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        buffer.append(")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitInnerJoinOperator(InnerJoinOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("join (").append(op.getCondition().getValue().accept(exprVisitor, indent))
+                .append(")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("left outer join (")
+                .append(op.getCondition().getValue().accept(exprVisitor, indent)).append(")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("nested tuple source");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitOrderOperator(OrderOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("order ");
+        for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
+            String fst;
+            switch (p.first.getKind()) {
+                case ASC: {
+                    fst = "ASC";
+                    break;
+                }
+                case DESC: {
+                    fst = "DESC";
+                    break;
+                }
+                default: {
+                    fst = p.first.getExpressionRef().toString();
+                }
+            }
+            buffer.append("(" + fst + ", " + p.second.getValue().accept(exprVisitor, indent) + ") ");
+        }
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitAssignOperator(AssignOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("assign ").append(op.getVariables()).append(" <- ");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitWriteOperator(WriteOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("write ");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitDistributeResultOperator(DistributeResultOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("distribute result ");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitWriteResultOperator(WriteResultOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("load ").append(op.getDataSource()).append(" from ")
+                .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)).append(" partitioned by ");
+        pprintExprList(op.getKeyExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitSelectOperator(SelectOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("select (").append(op.getCondition().getValue().accept(exprVisitor, indent))
+                .append(")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitProjectOperator(ProjectOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("project " + "(" + op.getVariables() + ")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitPartitioningSplitOperator(PartitioningSplitOperator op, Integer indent)
+            throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("partitioning-split (");
+        pprintExprList(op.getExpressions(), buffer, indent);
+        buffer.append(")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitSubplanOperator(SubplanOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("subplan {");
+        printNestedPlans(op, indent, buffer);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitUnionOperator(UnionAllOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("union");
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> v : op.getVariableMappings()) {
+            buffer.append(" (" + v.first + ", " + v.second + ", " + v.third + ")");
+        }
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitUnnestOperator(UnnestOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("unnest " + op.getVariable());
+        if (op.getPositionalVariable() != null) {
+            buffer.append(" at " + op.getPositionalVariable());
+        }
+        buffer.append(" <- " + op.getExpressionRef().getValue().accept(exprVisitor, indent));
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append(
+                "unnest-map " + op.getVariables() + " <- "
+                        + op.getExpressionRef().getValue().accept(exprVisitor, indent));
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitDataScanOperator(DataSourceScanOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append(
+                "data-scan " + op.getProjectVariables() + "<-" + op.getVariables() + " <- " + op.getDataSource());
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitLimitOperator(LimitOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("limit " + op.getMaxObjects().getValue().accept(exprVisitor, indent));
+        ILogicalExpression offset = op.getOffset().getValue();
+        if (offset != null) {
+            buffer.append(", " + offset.accept(exprVisitor, indent));
+        }
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitExchangeOperator(ExchangeOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("exchange ");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitScriptOperator(ScriptOperator op, Integer indent) {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append(
+                "script (in: " + op.getInputVariables() + ") (out: " + op.getOutputVariables() + ")");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitReplicateOperator(ReplicateOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("replicate ");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitMaterializeOperator(MaterializeOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("materialize ");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitInsertDeleteOperator(InsertDeleteOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
+        addIndent(buffer, indent).append(header).append(op.getDataSource()).append(" from ")
+                .append(op.getPayloadExpression().getValue().accept(exprVisitor, indent)).append(" partitioned by ");
+        pprintExprList(op.getPrimaryKeyExpressions(), buffer, indent);
+        if (op.isBulkload()) {
+            buffer.append(" [bulkload]");
+        }
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Integer indent)
+            throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
+        addIndent(buffer, indent).append(header).append(op.getIndexName()).append(" on ")
+                .append(op.getDataSourceIndex().getDataSource()).append(" from ");
+        pprintExprList(op.getSecondaryKeyExpressions(), buffer, indent);
+        if (op.isBulkload()) {
+            buffer.append(" [bulkload]");
+        }
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitTokenizeOperator(TokenizeOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("tokenize ").append(op.getTokenizeVars()).append(" <- ");
+        pprintExprList(op.getSecondaryKeyExpressions(), buffer, indent);
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitSinkOperator(SinkOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append("sink");
+        return buffer.toString();
+    }
+
+    @Override
+    public String visitExtensionOperator(ExtensionOperator op, Integer indent) throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append(op.toString());
+        return buffer.toString();
+    }
+
+    protected static final StringBuilder addIndent(StringBuilder buffer, int level) {
+        for (int i = 0; i < level; ++i) {
+            buffer.append(' ');
+        }
+        return buffer;
+    }
+
+    protected void printNestedPlans(AbstractOperatorWithNestedPlans op, Integer indent, StringBuilder buffer)
+            throws AlgebricksException {
+        boolean first = true;
+        if (op.getNestedPlans().isEmpty()) {
+            buffer.append("}");
+        } else {
+            for (ILogicalPlan p : op.getNestedPlans()) {
+                // PrettyPrintUtil.indent(buffer, level + 10).append("var " +
+                // p.first + ":\n");
+                buffer.append("\n");
+                if (first) {
+                    first = false;
+                } else {
+                    addIndent(buffer, indent).append("       {\n");
+                }
+                PlanPrettyPrinter.printPlan(p, buffer, this, indent + 10);
+                addIndent(buffer, indent).append("       }");
+            }
+        }
+    }
+
+    protected void pprintExprList(List<Mutable<ILogicalExpression>> expressions, StringBuilder buffer, Integer indent)
+            throws AlgebricksException {
+        buffer.append("[");
+        boolean first = true;
+        for (Mutable<ILogicalExpression> exprRef : expressions) {
+            if (first) {
+                first = false;
+            } else {
+                buffer.append(", ");
+            }
+            buffer.append(exprRef.getValue().accept(exprVisitor, indent));
+        }
+        buffer.append("]");
+    }
+
+    protected void pprintVeList(StringBuilder sb, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList,
+            Integer indent) throws AlgebricksException {
+        sb.append("[");
+        boolean fst = true;
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
+            if (fst) {
+                fst = false;
+            } else {
+                sb.append("; ");
+            }
+            if (ve.first != null) {
+                sb.append(ve.first + " := " + ve.second);
+            } else {
+                sb.append(ve.second.getValue().accept(exprVisitor, indent));
+            }
+        }
+        sb.append("]");
+    }
+
+    @Override
+    public String visitExternalDataLookupOperator(ExternalDataLookupOperator op, Integer indent)
+            throws AlgebricksException {
+        StringBuilder buffer = new StringBuilder();
+        addIndent(buffer, indent).append(
+                "external-instant-lookup " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
+        return buffer.toString();
+    }
+
+}

[46/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
deleted file mode 100644
index e0a2cf1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class InnerJoinOperator extends AbstractBinaryJoinOperator {
-
-    public InnerJoinOperator(Mutable<ILogicalExpression> condition) {
-        super(JoinKind.INNER, condition);
-    }
-
-    public InnerJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
-            Mutable<ILogicalOperator> input2) {
-        super(JoinKind.INNER, condition, input1, input2);
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.INNERJOIN;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitInnerJoinOperator(this, arg);
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
deleted file mode 100644
index 134506c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class InsertDeleteOperator extends AbstractLogicalOperator {
-
-    public enum Kind {
-        INSERT,
-        DELETE
-    }
-
-    private final IDataSource<?> dataSource;
-
-    private final Mutable<ILogicalExpression> payloadExpr;
-
-    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
-
-    private final Kind operation;
-    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
-
-    private final boolean bulkload;
-
-    public InsertDeleteOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payloadExpr,
-            List<Mutable<ILogicalExpression>> primaryKeyExprs, Kind operation, boolean bulkload) {
-        this.dataSource = dataSource;
-        this.payloadExpr = payloadExpr;
-        this.primaryKeyExprs = primaryKeyExprs;
-        this.operation = operation;
-        this.bulkload = bulkload;
-    }
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
-        boolean changed = false;
-        changed = transform.transform(payloadExpr);
-        for (Mutable<ILogicalExpression> e : primaryKeyExprs) {
-            changed |= transform.transform(e);
-        }
-        return changed;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitInsertDeleteOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.INSERT_DELETE;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
-        return primaryKeyExprs;
-    }
-
-    public IDataSource<?> getDataSource() {
-        return dataSource;
-    }
-
-    public Mutable<ILogicalExpression> getPayloadExpression() {
-        return payloadExpr;
-    }
-
-    public Kind getOperation() {
-        return operation;
-    }
-
-    public boolean isBulkload() {
-        return bulkload;
-	}
-
-    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
-        this.additionalFilteringExpressions = additionalFilteringExpressions;
-    }
-
-    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
-        return additionalFilteringExpressions;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
deleted file mode 100644
index 1504039..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class LeftOuterJoinOperator extends AbstractBinaryJoinOperator {
-
-    public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition) {
-        super(JoinKind.LEFT_OUTER, condition);
-    }
-
-    public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
-            Mutable<ILogicalOperator> input2) {
-        super(JoinKind.LEFT_OUTER, condition, input1, input2);
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.LEFTOUTERJOIN;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitLeftOuterJoinOperator(this, arg);
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        int n = inputs.size();
-        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
-        for (int i = 0; i < n; i++) {
-            envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
-        }
-        PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
-        List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(inputs.get(1).getValue(), liveVars); // live variables from outer branch can be null together
-        env.getCorrelatedNullableVariableLists().add(liveVars);
-        return env;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
deleted file mode 100644
index 79145f4..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class LimitOperator extends AbstractLogicalOperator {
-
-    private final Mutable<ILogicalExpression> maxObjects; // mandatory
-    private final Mutable<ILogicalExpression> offset; // optional
-    private boolean topmost;
-
-    public LimitOperator(ILogicalExpression maxObjectsExpr, ILogicalExpression offsetExpr, boolean topmost) {
-        this.maxObjects = new MutableObject<ILogicalExpression>(maxObjectsExpr);
-        this.offset = new MutableObject<ILogicalExpression>(offsetExpr);
-        this.topmost = topmost;
-    }
-
-    public LimitOperator(ILogicalExpression maxObjectsExpr, boolean topmost) {
-        this(maxObjectsExpr, null, topmost);
-    }
-
-    public LimitOperator(ILogicalExpression maxObjects, ILogicalExpression offset) {
-        this(maxObjects, offset, true);
-    }
-
-    public LimitOperator(ILogicalExpression maxObjects) {
-        this(maxObjects, null, true);
-    }
-
-    public Mutable<ILogicalExpression> getMaxObjects() {
-        return maxObjects;
-    }
-
-    public Mutable<ILogicalExpression> getOffset() {
-        return offset;
-    }
-
-    public boolean isTopmostLimitOp() {
-        return topmost;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitLimitOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean b = false;
-        if (visitor.transform(maxObjects)) {
-            b = true;
-        }
-        if (offset.getValue() != null) {
-            if (visitor.transform(offset)) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.LIMIT;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return true;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java
deleted file mode 100644
index 5cecf07..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/MaterializeOperator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class MaterializeOperator extends AbstractLogicalOperator {
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.MATERIALIZE;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitMaterializeOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
deleted file mode 100644
index f6f62b6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class NestedTupleSourceOperator extends AbstractLogicalOperator {
-    private Mutable<ILogicalOperator> dataSourceReference;
-
-    public NestedTupleSourceOperator(Mutable<ILogicalOperator> dataSourceReference) {
-        this.dataSourceReference = dataSourceReference;
-    }
-
-    public ILogicalOperator getSourceOperator() {
-        return dataSourceReference.getValue().getInputs().get(0).getValue();
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.NESTEDTUPLESOURCE;
-    }
-
-    public Mutable<ILogicalOperator> getDataSourceReference() {
-        return dataSourceReference;
-    }
-
-    public void setDataSourceReference(Mutable<ILogicalOperator> dataSourceReference) {
-        this.dataSourceReference = dataSourceReference;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        ILogicalOperator topOp = dataSourceReference.getValue();
-        for (Mutable<ILogicalOperator> i : topOp.getInputs()) {
-            schema.addAll(i.getValue().getSchema());
-        }
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
-        // do nothing
-        return false;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitNestedTupleSourceOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException {
-        ITypeEnvPointer[] p = new ITypeEnvPointer[1];
-        p[0] = new ITypeEnvPointer() {
-
-            @Override
-            public IVariableTypeEnvironment getTypeEnv() {
-                ILogicalOperator op = dataSourceReference.getValue().getInputs().get(0).getValue();
-                return ctx.getOutputTypeEnvironment(op);
-            }
-        };
-        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
-                ctx.getMetadataProvider(), TypePropagationPolicy.ALL, p);
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return computeOutputTypeEnvironment(ctx);
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
deleted file mode 100644
index 335fb36..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class OrderOperator extends AbstractLogicalOperator {
-
-    public interface IOrder {
-        public enum OrderKind {
-            FUNCTIONCALL,
-            ASC,
-            DESC
-        };
-
-        public Mutable<ILogicalExpression> getExpressionRef();
-
-        public OrderKind getKind();
-    }
-
-    public static IOrder ASC_ORDER = new IOrder() {
-
-        @Override
-        public Mutable<ILogicalExpression> getExpressionRef() {
-            return null;
-        }
-
-        @Override
-        public OrderKind getKind() {
-            return OrderKind.ASC;
-        }
-
-    };
-
-    public static IOrder DESC_ORDER = new IOrder() {
-
-        @Override
-        public Mutable<ILogicalExpression> getExpressionRef() {
-            return null;
-        }
-
-        @Override
-        public OrderKind getKind() {
-            return OrderKind.DESC;
-        }
-    };
-
-    public class FunOrder implements IOrder {
-        private final Mutable<ILogicalExpression> f;
-
-        public FunOrder(Mutable<ILogicalExpression> f) {
-            this.f = f;
-        }
-
-        @Override
-        public Mutable<ILogicalExpression> getExpressionRef() {
-            return f;
-        }
-
-        @Override
-        public OrderKind getKind() {
-            return OrderKind.FUNCTIONCALL;
-        }
-
-    };
-
-    private final List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions;
-
-    // These are pairs of type (comparison, expr) where comparison is
-    // ASC or DESC or a boolean function of arity 2 that can take as
-    // arguments results of expr.
-
-    public OrderOperator() {
-        orderExpressions = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
-    }
-
-    public OrderOperator(List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions) {
-        this.orderExpressions = orderExpressions;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.ORDER;
-    }
-
-    public List<Pair<IOrder, Mutable<ILogicalExpression>>> getOrderExpressions() {
-        return orderExpressions;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean b = false;
-        for (Pair<IOrder, Mutable<ILogicalExpression>> p : orderExpressions) {
-            if (p.first.getKind() == OrderKind.FUNCTIONCALL) {
-                FunOrder fo = (FunOrder) p.first;
-                Mutable<ILogicalExpression> r1 = fo.getExpressionRef();
-                if (visitor.transform(r1)) {
-                    b = true;
-                }
-            }
-            if (visitor.transform(p.second)) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitOrderOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
deleted file mode 100644
index a6c8f6f..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.PartitioningSplitOperatorDescriptor;
-
-/**
- * Partitions it's input based on a given list of expressions.
- * Each expression is assumed to return true/false,
- * and there is exactly one output branch per expression (optionally, plus one default branch).
- * For each input tuple, the expressions are evaluated one-by-one,
- * and the tuple is written to first output branch whose corresponding
- * expression evaluates to true.
- * If all expressions evaluate to false, then
- * the tuple is written to the default output branch, if any was given.
- * If no output branch was given, then such tuples are simply dropped.
- * Given N expressions there may be N or N+1 output branches because the default output branch may be separate from the regular output branches.
- */
-public class PartitioningSplitOperator extends AbstractLogicalOperator {
-
-    private final List<Mutable<ILogicalExpression>> expressions;
-    private final int defaultBranchIndex;
-
-    public PartitioningSplitOperator(List<Mutable<ILogicalExpression>> expressions, int defaultBranchIndex) throws AlgebricksException {
-        this.expressions = expressions;
-        this.defaultBranchIndex = defaultBranchIndex;
-        // Check that the default output branch index is in [0, N], where N is the number of expressions.
-        if (defaultBranchIndex != PartitioningSplitOperatorDescriptor.NO_DEFAULT_BRANCH
-                && defaultBranchIndex > expressions.size()) {
-            throw new AlgebricksException("Default branch index out of bounds. Number of exprs given: "
-                    + expressions.size() + ". The maximum default branch index may therefore be: " + expressions.size());
-        }
-    }
-
-    public List<Mutable<ILogicalExpression>> getExpressions() {
-        return expressions;
-    }
-
-    public int getDefaultBranchIndex() {
-        return defaultBranchIndex;
-    }
-    
-    public int getNumOutputBranches() {
-        return (defaultBranchIndex == expressions.size()) ? expressions.size() + 1 : expressions.size();
-    }
-    
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.PARTITIONINGSPLIT;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean b = false;
-        for (int i = 0; i < expressions.size(); i++) {
-            if (visitor.transform(expressions.get(i))) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitPartitioningSplitOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
deleted file mode 100644
index bcba280..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FilteredVariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ProjectOperator extends AbstractLogicalOperator {
-
-    private final List<LogicalVariable> variables;
-
-    public ProjectOperator(List<LogicalVariable> variables) {
-        this.variables = variables;
-    }
-
-    public ProjectOperator(LogicalVariable v) {
-        this.variables = new ArrayList<LogicalVariable>(1);
-        this.getVariables().add(v);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
-        // do nothing
-        return false;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.PROJECT;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(getVariables());
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new FilteredVariablePropagationPolicy(getVariables());
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitProjectOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return true;
-    }
-
-    public List<LogicalVariable> getVariables() {
-        return variables;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
deleted file mode 100644
index 2c5bd54..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ReplicateOperator extends AbstractLogicalOperator {
-
-    private int outputArity = 2;
-    private boolean[] outputMaterializationFlags = new boolean[outputArity];
-    private List<Mutable<ILogicalOperator>> outputs;
-
-    public ReplicateOperator(int outputArity) {
-        this.outputArity = outputArity;
-        this.outputMaterializationFlags = new boolean[outputArity];
-        this.outputs = new ArrayList<Mutable<ILogicalOperator>>();
-    }
-
-    public ReplicateOperator(int outputArity, boolean[] outputMaterializationFlags) {
-        this.outputArity = outputArity;
-        this.outputMaterializationFlags = outputMaterializationFlags;
-        this.outputs = new ArrayList<Mutable<ILogicalOperator>>();
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.REPLICATE;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitReplicateOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
-    }
-
-    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
-        // do nothing
-    }
-
-    public int getOutputArity() {
-        return outputArity;
-    }
-
-    public int setOutputArity(int outputArity) {
-        return this.outputArity = outputArity;
-    }
-
-    public void setOutputMaterializationFlags(boolean[] outputMaterializationFlags) {
-        this.outputMaterializationFlags = outputMaterializationFlags;
-    }
-
-    public boolean[] getOutputMaterializationFlags() {
-        return outputMaterializationFlags;
-    }
-
-    public List<Mutable<ILogicalOperator>> getOutputs() {
-        return outputs;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        return createPropagatingAllInputsTypeEnvironment(ctx);
-    }
-
-    public boolean isBlocker() {
-        for (boolean requiresMaterialization : outputMaterializationFlags) {
-            if (requiresMaterialization) {
-                return true;
-            }
-        }
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
deleted file mode 100644
index 4f608bb..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class RunningAggregateOperator extends AbstractAssignOperator {
-
-    public RunningAggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
-        super(variables, expressions);
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.RUNNINGAGGREGATE;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitRunningAggregateOperator(this, arg);
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                target.addAllVariables(sources[0]);
-                for (LogicalVariable v : variables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            env.setVarType(
-                    variables.get(i),
-                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
-                            env));
-        }
-        return env;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
deleted file mode 100644
index 46e9859..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ScriptOperator extends AbstractLogicalOperator {
-
-    private ArrayList<LogicalVariable> inputVariables;
-    private ArrayList<LogicalVariable> outputVariables;
-    private IScriptDescription scriptDesc;
-
-    public ScriptOperator(IScriptDescription scriptDesc, ArrayList<LogicalVariable> inputVariables,
-            ArrayList<LogicalVariable> outputVariables) {
-        this.inputVariables = inputVariables;
-        this.outputVariables = outputVariables;
-        this.scriptDesc = scriptDesc;
-    }
-
-    public ArrayList<LogicalVariable> getInputVariables() {
-        return inputVariables;
-    }
-
-    public ArrayList<LogicalVariable> getOutputVariables() {
-        return outputVariables;
-    }
-
-    public IScriptDescription getScriptDescription() {
-        return scriptDesc;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.SCRIPT;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitScriptOperator(this, arg);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                for (LogicalVariable v : outputVariables) {
-                    target.addVariable(v);
-                }
-            }
-        };
-
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        this.schema = outputVariables;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-                ctx.getMetadataProvider());
-        for (Pair<LogicalVariable, Object> p : scriptDesc.getVarTypePairs()) {
-            env.setVarType(p.first, p.second);
-        }
-        return env;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
deleted file mode 100644
index 5aa229c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SelectOperator extends AbstractLogicalOperator {
-    private final Mutable<ILogicalExpression> condition;
-    private final boolean retainNull;
-    private final LogicalVariable nullPlaceholderVar;
-
-    public SelectOperator(Mutable<ILogicalExpression> condition, boolean retainNull, LogicalVariable nullPlaceholderVar) {
-        this.condition = condition;
-        this.retainNull = retainNull;
-        this.nullPlaceholderVar = nullPlaceholderVar;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.SELECT;
-    }
-
-    public Mutable<ILogicalExpression> getCondition() {
-        return condition;
-    }
-
-    public boolean getRetainNull() {
-        return retainNull;
-    }
-
-    public LogicalVariable getNullPlaceholderVariable() throws AlgebricksException {
-        return nullPlaceholderVar;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return visitor.transform(condition);
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitSelectOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return true;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[1];
-        envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
-        PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
-        if (condition.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) condition.getValue();
-            if (f1.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.NOT)) {
-                ILogicalExpression a1 = f1.getArguments().get(0).getValue();
-                if (a1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                    AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
-                    if (f2.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_NULL)) {
-                        ILogicalExpression a2 = f2.getArguments().get(0).getValue();
-                        if (a2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                            LogicalVariable var = ((VariableReferenceExpression) a2).getVariableReference();
-                            env.getNonNullVariables().add(var);
-                        }
-                    }
-                }
-            }
-        }
-        return env;
-    }
-
-    @Override
-    public boolean requiresVariableReferenceExpressions() {
-        return false;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
deleted file mode 100644
index 84ad33b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SinkOperator extends AbstractLogicalOperator {
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>();
-        for (int i = 0; i < inputs.size(); i++) {
-            for (LogicalVariable v : inputs.get(i).getValue().getSchema()) {
-                if (!schema.contains(v))
-                	schema.add(v);
-            }
-
-        }
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitSinkOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ALL;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-      ITypeEnvPointer[] envPointers = new ITypeEnvPointer[inputs.size()];
-      for (int i = 0; i < inputs.size(); i++) {
-          envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
-      }
-      PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-              ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
-      return env;
-
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.SINK;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
deleted file mode 100644
index c0a01b3..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SubplanOperator extends AbstractOperatorWithNestedPlans {
-
-    public SubplanOperator() {
-        super();
-    }
-
-    public SubplanOperator(List<ILogicalPlan> plans) {
-        super(plans);
-    }
-
-    public SubplanOperator(ILogicalOperator planRoot) {
-        ArrayList<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>(1);
-        roots.add(new MutableObject<ILogicalOperator>(planRoot));
-        nestedPlans.add(new ALogicalPlanImpl(roots));
-    }
-
-    public void setRootOp(Mutable<ILogicalOperator> opRef) {
-        ILogicalPlan p = new ALogicalPlanImpl(opRef);
-        nestedPlans.add(p);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
-        // do nothing
-        return false;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.SUBPLAN;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.ADDNEWVARIABLES;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitSubplanOperator(this, arg);
-    }
-
-    @Override
-    public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
-        // do nothing
-    }
-
-    @Override
-    public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
-        // do nothing
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        int n = 0;
-        for (ILogicalPlan p : nestedPlans) {
-            n += p.getRoots().size();
-        }
-        ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n + 1];
-        envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
-        int i = 1;
-        for (ILogicalPlan p : nestedPlans) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
-                i++;
-            }
-        }
-        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
-                ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java
deleted file mode 100644
index 82c1035..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/TokenizeOperator.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class TokenizeOperator extends AbstractLogicalOperator {
-
-    private final IDataSourceIndex<?, ?> dataSourceIndex;
-    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
-    private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
-    // logical variables for token, and number of token
-    private final List<LogicalVariable> tokenizeVars;
-    private final Mutable<ILogicalExpression> filterExpr;
-    private final Kind operation;
-    private final boolean bulkload;
-    private final boolean isPartitioned;
-    // contains the type for each variable in the tokenizeVars
-    private final List<Object> tokenizeVarTypes;
-    private List<Mutable<ILogicalExpression>> additionalFilteringExpressions;
-
-    public TokenizeOperator(IDataSourceIndex<?, ?> dataSourceIndex,
-            List<Mutable<ILogicalExpression>> primaryKeyExprs,
-            List<Mutable<ILogicalExpression>> secondaryKeyExprs,
-            List<LogicalVariable> tokenizeVars,
-            Mutable<ILogicalExpression> filterExpr, Kind operation,
-            boolean bulkload, boolean isPartitioned,
-            List<Object> tokenizeVarTypes) {
-        this.dataSourceIndex = dataSourceIndex;
-        this.primaryKeyExprs = primaryKeyExprs;
-        this.secondaryKeyExprs = secondaryKeyExprs;
-        this.tokenizeVars = tokenizeVars;
-        this.filterExpr = filterExpr;
-        this.operation = operation;
-        this.bulkload = bulkload;
-        this.isPartitioned = isPartitioned;
-        this.tokenizeVarTypes = tokenizeVarTypes;
-    }
-
-    @Override
-    public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-        schema.addAll(tokenizeVars);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(
-            ILogicalExpressionReferenceTransform visitor)
-            throws AlgebricksException {
-        boolean b = false;
-        for (int i = 0; i < primaryKeyExprs.size(); i++) {
-            if (visitor.transform(primaryKeyExprs.get(i))) {
-                b = true;
-            }
-        }
-        for (int i = 0; i < secondaryKeyExprs.size(); i++) {
-            if (visitor.transform(secondaryKeyExprs.get(i))) {
-                b = true;
-            }
-        }
-        return b;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg)
-            throws AlgebricksException {
-        return visitor.visitTokenizeOperator(this, arg);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target,
-                    IOperatorSchema... sources) throws AlgebricksException {
-                target.addAllVariables(sources[0]);
-                for (LogicalVariable v : tokenizeVars) {
-                    target.addVariable(v);
-                }
-            }
-        };
-
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.TOKENIZE;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(
-            ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
-
-        // If the secondary index is not length-partitioned, create one new
-        // output variable - token.
-        // If the secondary index is length-partitioned, create two new output
-        // variables - token, number of token.
-        // The type of this variable will be the same as the type of the
-        // secondary key. If the secondary is list type, then the element type
-        // of the list.
-        // Along with PK, the tokenizer will generate [token, number of tokens,
-        // PK] pairs.
-
-        for (int i = 0; i < tokenizeVars.size(); i++) {
-            env.setVarType(tokenizeVars.get(i), tokenizeVarTypes.get(i));
-        }
-
-        return env;
-    }
-
-    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
-        return primaryKeyExprs;
-    }
-
-    public IDataSourceIndex<?, ?> getDataSourceIndex() {
-        return dataSourceIndex;
-    }
-
-    public List<Mutable<ILogicalExpression>> getSecondaryKeyExpressions() {
-        return secondaryKeyExprs;
-    }
-
-    public List<LogicalVariable> getTokenizeVars() {
-        return tokenizeVars;
-    }
-
-    public Mutable<ILogicalExpression> getFilterExpression() {
-        return filterExpr;
-    }
-
-    public Kind getOperation() {
-        return operation;
-    }
-
-    public boolean isBulkload() {
-        return bulkload;
-    }
-
-    public boolean isPartitioned() {
-        return isPartitioned;
-    }
-
-    public List<Object> getTokenizeVarTypes() {
-        return tokenizeVarTypes;
-    }
-
-    public void setAdditionalFilteringExpressions(List<Mutable<ILogicalExpression>> additionalFilteringExpressions) {
-        this.additionalFilteringExpressions = additionalFilteringExpressions;
-    }
-
-    public List<Mutable<ILogicalExpression>> getAdditionalFilteringExpressions() {
-        return additionalFilteringExpressions;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
deleted file mode 100644
index 97a77ec..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class UnionAllOperator extends AbstractLogicalOperator {
-
-    // (left-var, right-var, out-var)
-    private List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap;
-
-    public UnionAllOperator(List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap) {
-        this.varMap = varMap;
-    }
-
-    public List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> getVariableMappings() {
-        return varMap;
-    }
-
-    @Override
-    public LogicalOperatorTag getOperatorTag() {
-        return LogicalOperatorTag.UNIONALL;
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return new VariablePropagationPolicy() {
-
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
-                    target.addVariable(t.third);
-                }
-            }
-        };
-
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitUnionOperator(this, arg);
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        for (LogicalVariable v1 : inputs.get(0).getValue().getSchema()) {
-            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
-                if (t.first.equals(v1)) {
-                    schema.add(t.third);
-                } else {
-                    schema.add(v1);
-                }
-            }
-        }
-        for (LogicalVariable v2 : inputs.get(1).getValue().getSchema()) {
-            for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
-                if (t.second.equals(v2)) {
-                    schema.add(t.third);
-                } else {
-                    schema.add(v2);
-                }
-            }
-        }
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-
-    @Override
-    public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
-                ctx.getMetadataProvider());
-        IVariableTypeEnvironment envLeft = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
-        if (envLeft == null) {
-            throw new AlgebricksException("Left input types for union operator are not computed.");
-        }
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
-            Object t1 = envLeft.getVarType(t.first);
-            if (t1 == null) {
-                throw new AlgebricksException("Failed typing union operator: no type for variable " + t.first);
-            }
-            env.setVarType(t.third, t1);
-        }
-        return env;
-    }
-
-}

[07/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
new file mode 100644
index 0000000..6390efc
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
@@ -0,0 +1,253 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Replaces variable reference expressions with their assigned function-call expression where applicable
+ * (some variables are generated by datasources).
+ * Inlining variables may enable other optimizations by allowing selects and assigns to be moved
+ * (e.g., a select may be pushed into a join to enable an efficient physical join operator).
+ * Preconditions/Assumptions:
+ * Assumes no projects are in the plan. Only inlines variables whose assigned expression is a function call
+ * (i.e., this rule ignores right-hand side constants and other variable references expressions
+ * Postconditions/Examples:
+ * All qualifying variables have been inlined.
+ * Example (simplified):
+ * Before plan:
+ * select <- [$$1 < $$2 + $$0]
+ * assign [$$2] <- [funcZ() + $$0]
+ * assign [$$0, $$1] <- [funcX(), funcY()]
+ * After plan:
+ * select <- [funcY() < funcZ() + funcX() + funcX()]
+ * assign [$$2] <- [funcZ() + funcX()]
+ * assign [$$0, $$1] <- [funcX(), funcY()]
+ */
+public class InlineVariablesRule implements IAlgebraicRewriteRule {
+
+    // Map of variables that could be replaced by their producing expression.
+    // Populated during the top-down sweep of the plan.
+    protected Map<LogicalVariable, ILogicalExpression> varAssignRhs = new HashMap<LogicalVariable, ILogicalExpression>();
+
+    // Visitor for replacing variable reference expressions with their originating expression.
+    protected InlineVariablesVisitor inlineVisitor = new InlineVariablesVisitor(varAssignRhs);
+
+    // Set of FunctionIdentifiers that we should not inline.
+    protected Set<FunctionIdentifier> doNotInlineFuncs = new HashSet<FunctionIdentifier>();
+
+    protected boolean hasRun = false;
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (hasRun) {
+            return false;
+        }
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+            return false;
+        }
+        prepare(context);
+        boolean modified = inlineVariables(opRef, context);
+        if (performFinalAction()) {
+            modified = true;
+        }
+        hasRun = true;
+        return modified;
+    }
+
+    protected void prepare(IOptimizationContext context) {
+        varAssignRhs.clear();
+        inlineVisitor.setContext(context);
+    }
+
+    protected boolean performBottomUpAction(AbstractLogicalOperator op) throws AlgebricksException {
+        // Only inline variables in operators that can deal with arbitrary expressions.
+        if (!op.requiresVariableReferenceExpressions()) {
+            inlineVisitor.setOperator(op);
+            return op.acceptExpressionTransform(inlineVisitor);
+        }
+        return false;
+    }
+
+    protected boolean performFinalAction() throws AlgebricksException {
+        return false;
+    }
+
+    protected boolean inlineVariables(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+        // Update mapping from variables to expressions during top-down traversal.
+        if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+            AssignOperator assignOp = (AssignOperator) op;
+            List<LogicalVariable> vars = assignOp.getVariables();
+            List<Mutable<ILogicalExpression>> exprs = assignOp.getExpressions();
+            for (int i = 0; i < vars.size(); i++) {
+                ILogicalExpression expr = exprs.get(i).getValue();
+                // Ignore functions that are either in the doNotInline set or are non-functional               
+                if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                    AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+                    if (doNotInlineFuncs.contains(funcExpr.getFunctionIdentifier()) || !funcExpr.isFunctional()) {
+                        continue;
+                    }
+                }
+                varAssignRhs.put(vars.get(i), exprs.get(i).getValue());
+            }
+        }
+
+        // Descend into children removing projects on the way.
+        boolean modified = false;
+        for (Mutable<ILogicalOperator> inputOpRef : op.getInputs()) {
+            if (inlineVariables(inputOpRef, context)) {
+                modified = true;
+            }
+        }
+
+        // Descend into subplan
+        if (op.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+            ALogicalPlanImpl subPlan = (ALogicalPlanImpl) ((SubplanOperator) op).getNestedPlans().get(0);
+            Mutable<ILogicalOperator> subPlanRootOpRef = subPlan.getRoots().get(0);
+            if (inlineVariables(subPlanRootOpRef, context)) {
+                modified = true;
+            }
+        }
+
+        if (performBottomUpAction(op)) {
+            modified = true;
+        }
+
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(op);
+            context.addToDontApplySet(this, op);
+            // Re-enable rules that we may have already tried. They could be applicable now after inlining.
+            context.removeFromAlreadyCompared(opRef.getValue());
+        }
+
+        return modified;
+    }
+
+    protected class InlineVariablesVisitor implements ILogicalExpressionReferenceTransform {
+
+        private final Map<LogicalVariable, ILogicalExpression> varAssignRhs;
+        private final Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
+        private final List<LogicalVariable> rhsUsedVars = new ArrayList<LogicalVariable>();
+        private ILogicalOperator op;
+        private IOptimizationContext context;
+        // If set, only replace this variable reference.
+        private LogicalVariable targetVar;
+
+        public InlineVariablesVisitor(Map<LogicalVariable, ILogicalExpression> varAssignRhs) {
+            this.varAssignRhs = varAssignRhs;
+        }
+
+        public void setTargetVariable(LogicalVariable targetVar) {
+            this.targetVar = targetVar;
+        }
+
+        public void setContext(IOptimizationContext context) {
+            this.context = context;
+        }
+
+        public void setOperator(ILogicalOperator op) throws AlgebricksException {
+            this.op = op;
+            liveVars.clear();
+        }
+
+        @Override
+        public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+            ILogicalExpression e = exprRef.getValue();
+            switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+                case VARIABLE: {
+                    LogicalVariable var = ((VariableReferenceExpression) e).getVariableReference();
+                    // Restrict replacement to targetVar if it has been set.
+                    if (targetVar != null && var != targetVar) {
+                        return false;
+                    }
+
+                    // Make sure has not been excluded from inlining.
+                    if (context.shouldNotBeInlined(var)) {
+                        return false;
+                    }
+
+                    ILogicalExpression rhs = varAssignRhs.get(var);
+                    if (rhs == null) {
+                        // Variable was not produced by an assign.
+                        return false;
+                    }
+
+                    // Make sure used variables from rhs are live.
+                    if (liveVars.isEmpty()) {
+                        VariableUtilities.getLiveVariables(op, liveVars);
+                    }
+                    rhsUsedVars.clear();
+                    rhs.getUsedVariables(rhsUsedVars);
+                    for (LogicalVariable rhsUsedVar : rhsUsedVars) {
+                        if (!liveVars.contains(rhsUsedVar)) {
+                            return false;
+                        }
+                    }
+
+                    // Replace variable reference with a clone of the rhs expr.
+                    exprRef.setValue(rhs.cloneExpression());
+                    return true;
+                }
+                case FUNCTION_CALL: {
+                    AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
+                    boolean modified = false;
+                    for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
+                        if (transform(arg)) {
+                            modified = true;
+                        }
+                    }
+                    return modified;
+                }
+                default: {
+                    return false;
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
new file mode 100644
index 0000000..c5aee3c
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.Iterator;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InsertOuterJoinRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op0;
+
+        Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
+        ILogicalPlan p = null;
+        while (plansIter.hasNext()) {
+            p = plansIter.next();
+        }
+        if (p == null) {
+            return false;
+        }
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
+        Mutable<ILogicalOperator> opUnder = subplan.getInputs().get(0);
+
+        if (OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) opUnder.getValue())) {
+            return false;
+        }
+
+        switch (op1.getOperatorTag()) {
+            case INNERJOIN: {
+                InnerJoinOperator join = (InnerJoinOperator) op1;
+                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
+                Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
+                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
+                if (ntsRef == null) {
+                    ntsRef = getNtsAtEndOfPipeline(rightRef);
+                    if (ntsRef == null) {
+                        return false;
+                    } else {
+                        Mutable<ILogicalOperator> t = leftRef;
+                        leftRef = rightRef;
+                        rightRef = t;
+                    }
+                }
+                ntsRef.setValue(opUnder.getValue());
+                LeftOuterJoinOperator loj = new LeftOuterJoinOperator(join.getCondition());
+                loj.getInputs().add(leftRef);
+                loj.getInputs().add(rightRef);
+                opRef.setValue(loj);
+                context.computeAndSetTypeEnvironmentForOperator(loj);
+                return true;
+            }
+            case LEFTOUTERJOIN: {
+                LeftOuterJoinOperator join = (LeftOuterJoinOperator) op1;
+                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
+                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
+                if (ntsRef == null) {
+                    return false;
+                }
+                ntsRef.setValue(opUnder.getValue());
+                opRef.setValue(join);
+                context.computeAndSetTypeEnvironmentForOperator(join);
+                return true;
+            }
+            default: {
+                return false;
+            }
+        }
+    }
+
+    private Mutable<ILogicalOperator> getNtsAtEndOfPipeline(Mutable<ILogicalOperator> opRef) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            return opRef;
+        }
+        if (op.getInputs().size() != 1) {
+            return null;
+        }
+        return getNtsAtEndOfPipeline(op.getInputs().get(0));
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
new file mode 100644
index 0000000..80b8815
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InsertProjectBeforeUnionRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        return false;
+    }
+
+    /**
+     * When the input schema to WriteOperator is different from the output
+     * schema in terms of variable order, add a project operator to get the
+     * write order
+     */
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
+            return false;
+        }
+        UnionAllOperator opUnion = (UnionAllOperator) op;
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = opUnion.getVariableMappings();
+        ArrayList<LogicalVariable> usedVariablesFromOne = new ArrayList<LogicalVariable>();
+        ArrayList<LogicalVariable> usedVariablesFromTwo = new ArrayList<LogicalVariable>();
+
+        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple : varMap) {
+            usedVariablesFromOne.add(triple.first);
+            usedVariablesFromTwo.add(triple.second);
+        }
+
+        ArrayList<LogicalVariable> inputSchemaOne = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(opUnion.getInputs().get(0).getValue(), inputSchemaOne);
+
+        ArrayList<LogicalVariable> inputSchemaTwo = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(opUnion.getInputs().get(1).getValue(), inputSchemaTwo);
+
+        boolean rewritten = false;
+        if (!isIdentical(usedVariablesFromOne, inputSchemaOne)) {
+            insertProjectOperator(opUnion, 0, usedVariablesFromOne, context);
+            rewritten = true;
+        }
+        if (!isIdentical(usedVariablesFromTwo, inputSchemaTwo)) {
+            insertProjectOperator(opUnion, 1, usedVariablesFromTwo, context);
+            rewritten = true;
+        }
+        return rewritten;
+    }
+
+    private void insertProjectOperator(UnionAllOperator opUnion, int branch, ArrayList<LogicalVariable> usedVariables,
+            IOptimizationContext context) throws AlgebricksException {
+        ProjectOperator projectOp = new ProjectOperator(usedVariables);
+        ILogicalOperator parentOp = opUnion.getInputs().get(branch).getValue();
+        projectOp.getInputs().add(new MutableObject<ILogicalOperator>(parentOp));
+        opUnion.getInputs().get(branch).setValue(projectOp);
+        projectOp.setPhysicalOperator(new StreamProjectPOperator());
+        context.computeAndSetTypeEnvironmentForOperator(projectOp);
+        context.computeAndSetTypeEnvironmentForOperator(parentOp);
+    }
+
+    private boolean isIdentical(List<LogicalVariable> finalSchema, List<LogicalVariable> inputSchema)
+            throws AlgebricksException {
+        int finalSchemaSize = finalSchema.size();
+        int inputSchemaSize = inputSchema.size();
+        if (finalSchemaSize != inputSchemaSize) {
+            return false;
+        }
+        for (int i = 0; i < finalSchemaSize; i++) {
+            LogicalVariable var1 = finalSchema.get(i);
+            LogicalVariable var2 = inputSchema.get(i);
+            if (!var1.equals(var2))
+                return false;
+        }
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
new file mode 100644
index 0000000..51f2025
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IntroHashPartitionMergeExchange implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getPhysicalOperator() == null
+                || (op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_EXCHANGE && op1
+                        .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE)) {
+            return false;
+        }
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+        if (op2.getPhysicalOperator() == null
+                || op2.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.SORT_MERGE_EXCHANGE) {
+            return false;
+        }
+        if (op1.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE) {
+            // if it is a hash_partition_merge_exchange, the sort_merge_exchange can be simply removed
+            op1.getInputs().get(0).setValue(op2.getInputs().get(0).getValue());
+            op1.computeDeliveredPhysicalProperties(context);
+            return true;
+        }
+        HashPartitionExchangePOperator hpe = (HashPartitionExchangePOperator) op1.getPhysicalOperator();
+        SortMergeExchangePOperator sme = (SortMergeExchangePOperator) op2.getPhysicalOperator();
+        List<OrderColumn> ocList = new ArrayList<OrderColumn>();
+        for (OrderColumn oc : sme.getSortColumns()) {
+            ocList.add(oc);
+        }
+        HashPartitionMergeExchangePOperator hpme = new HashPartitionMergeExchangePOperator(ocList, hpe.getHashFields(),
+                hpe.getDomain());
+        op1.setPhysicalOperator(hpme);
+        op1.getInputs().get(0).setValue(op2.getInputs().get(0).getValue());
+        op1.computeDeliveredPhysicalProperties(context);
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
new file mode 100644
index 0000000..8929623
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.ListIterator;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+
+public class IntroJoinInsideSubplanRule extends AbstractDecorrelationRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op0;
+
+        Mutable<ILogicalOperator> leftRef = subplan.getInputs().get(0);
+        if (((AbstractLogicalOperator) leftRef.getValue()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+            return false;
+        }
+
+        ListIterator<ILogicalPlan> plansIter = subplan.getNestedPlans().listIterator();
+        ILogicalPlan p = null;
+        while (plansIter.hasNext()) {
+            p = plansIter.next();
+        }
+        if (p == null) {
+            return false;
+        }
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef1 = p.getRoots().get(0);
+
+        while (true) {
+            AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getValue();
+            if (op1.getInputs().size() != 1) {
+                return false;
+            }
+            if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
+                Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+                if (op2.getOperatorTag() != LogicalOperatorTag.SELECT && descOrSelfIsScanOrJoin(op2)) {
+                    Set<LogicalVariable> free2 = new HashSet<LogicalVariable>();
+                    OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, free2);
+                    if (free2.isEmpty()) {
+                        Set<LogicalVariable> free1 = new HashSet<LogicalVariable>();
+                        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op1, free1);
+                        if (!free1.isEmpty()) {
+                            OperatorManipulationUtil.ntsToEts(op2Ref, context);
+                            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(
+                                    new MutableObject<ILogicalOperator>(subplan));
+                            Mutable<ILogicalOperator> ntsRef = new MutableObject<ILogicalOperator>(nts);
+                            Mutable<ILogicalOperator> innerRef = new MutableObject<ILogicalOperator>(op2);
+                            InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(
+                                    ConstantExpression.TRUE), ntsRef, innerRef);
+                            op2Ref.setValue(join);
+                            context.computeAndSetTypeEnvironmentForOperator(nts);
+                            context.computeAndSetTypeEnvironmentForOperator(join);
+                            return true;
+                        }
+                    }
+                }
+            }
+            opRef1 = op1.getInputs().get(0);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
new file mode 100644
index 0000000..b0dbb1e
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+
+public class IntroduceAggregateCombinerRule extends AbstractIntroduceCombinerRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (context.checkIfInDontApplySet(this, op)) {
+            return false;
+        }
+        context.addToDontApplySet(this, op);
+        if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AggregateOperator aggOp = (AggregateOperator) op;
+        if (!aggOp.isGlobal() || aggOp.getExecutionMode() == ExecutionMode.LOCAL) {
+            return false;
+        }
+        Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
+        Pair<Boolean, Mutable<ILogicalOperator>> result = tryToPushAgg(aggOp, null, toReplaceSet, context);
+        if (!result.first || result.second == null) {
+            return false;
+        }
+        replaceOriginalAggFuncs(toReplaceSet);
+        context.computeAndSetTypeEnvironmentForOperator(aggOp);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
new file mode 100644
index 0000000..aa418ea
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+
+public class IntroduceGroupByCombinerRule extends AbstractIntroduceGroupByCombinerRule {
+
+    @Override
+    protected void processNullTest(IOptimizationContext context, GroupByOperator nestedGby,
+            List<LogicalVariable> aggregateVarsProducedByCombiner) {
+        /** The default introduce group-by combiner rule ignores null test, however a language implementor can decide their own semantics. */
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
new file mode 100644
index 0000000..a6d9ab0
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
+
+/**
+ * The rule searches for SUBPLAN operator with a optional PROJECT operator and
+ * an AGGREGATE followed by a join operator.
+ *
+ * <pre>
+ * Before
+ * 
+ *   plan__parent
+ *   SUBPLAN {
+ *     PROJECT?
+ *     AGGREGATE
+ *     plan__nested_A
+ *     INNER_JOIN | LEFT_OUTER_JOIN ($condition, $left, $right)
+ *       plan__nested_B
+ *   }
+ *   plan__child
+ * 
+ *   where $condition does not equal a constant true.
+ * 
+ * After (This is a general application of the rule, specifics may vary based on the query plan.)
+ * 
+ *   plan__parent
+ *   GROUP_BY {
+ *     PROJECT?
+ *     AGGREGATE
+ *     plan__nested_A
+ *     SELECT( algebricks:not( is_null( $right ) ) )
+ *     NESTED_TUPLE_SOURCE
+ *   }
+ *   SUBPLAN {
+ *     INNER_JOIN | LEFT_OUTER_JOIN ($condition, $left, $right)
+ *       plan__nested_B
+ *   }
+ *   plan__child
+ * </pre>
+ *
+ * @author prestonc
+ */
+
+public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op0;
+
+        Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
+        ILogicalPlan p = null;
+        while (plansIter.hasNext()) {
+            p = plansIter.next();
+        }
+        if (p == null) {
+            return false;
+        }
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
+
+        Mutable<ILogicalOperator> botRef = subplanRoot;
+        AbstractLogicalOperator op2;
+        // Project is optional
+        if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+            op2 = op1;
+        } else {
+            ProjectOperator project = (ProjectOperator) op1;
+            botRef = project.getInputs().get(0);
+            op2 = (AbstractLogicalOperator) botRef.getValue();
+        }
+        if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AggregateOperator aggregate = (AggregateOperator) op2;
+
+        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+        VariableUtilities.getUsedVariables(aggregate, free);
+
+        Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
+
+        while (op3.getInputs().size() == 1) {
+            Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
+            VariableUtilities.getProducedVariables(op3, prod);
+            free.removeAll(prod);
+            VariableUtilities.getUsedVariables(op3, free);
+            botRef = op3Ref;
+            op3Ref = op3.getInputs().get(0);
+            op3 = (AbstractLogicalOperator) op3Ref.getValue();
+        }
+
+        if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+                && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
+        if (join.getCondition().getValue() == ConstantExpression.TRUE) {
+            return false;
+        }
+        VariableUtilities.getUsedVariables(join, free);
+
+        AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
+        // see if there's an NTS at the end of the pipeline
+        NestedTupleSourceOperator outerNts = getNts(b0);
+        if (outerNts == null) {
+            AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
+            outerNts = getNts(b1);
+            if (outerNts == null) {
+                return false;
+            }
+        }
+
+        Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
+        if (pkVars == null || pkVars.size() < 1) {
+            // there is no non-trivial primary key, group-by keys are all live variables
+            ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
+            pkVars = new HashSet<LogicalVariable>();
+            VariableUtilities.getLiveVariables(subplanInput, pkVars);
+        }
+        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
+
+        Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
+        LogicalVariable testForNull = null;
+        AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
+        switch (right.getOperatorTag()) {
+            case UNNEST: {
+                UnnestOperator innerUnnest = (UnnestOperator) right;
+                // Select [ $y != null ]
+                testForNull = innerUnnest.getVariable();
+                break;
+            }
+            case RUNNINGAGGREGATE: {
+                ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
+                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
+                VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
+                if (!producedVars.isEmpty()) {
+                    // Select [ $y != null ]
+                    testForNull = producedVars.iterator().next();
+                }
+                break;
+            }
+            case DATASOURCESCAN: {
+                DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
+                // Select [ $y != null ]
+                if (innerScan.getVariables().size() == 1) {
+                    testForNull = innerScan.getVariables().get(0);
+                }
+                break;
+            }
+        }
+        if (testForNull == null) {
+            testForNull = context.newVar();
+            AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(
+                    ConstantExpression.TRUE));
+            tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
+            rightRef.setValue(tmpAsgn);
+            context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
+        }
+
+        IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_NULL);
+        ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq,
+                new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
+        IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
+        ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot,
+                new MutableObject<ILogicalExpression>(isNullTest));
+        SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false,
+                null);
+        GroupByOperator g = new GroupByOperator();
+        Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
+        NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
+        opRef.setValue(g);
+        selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
+
+        List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
+        prodInpList.clear();
+        prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
+
+        ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
+        g.getNestedPlans().add(gPlan);
+        subplanRoot.setValue(op3Ref.getValue());
+        g.getInputs().add(newSubplanRef);
+
+        HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
+        VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
+        underVars.removeAll(pkVars);
+        Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
+        context.updatePrimaryKeys(mappedVars);
+        for (LogicalVariable uv : underVars) {
+            g.getDecorList().add(
+                    new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(
+                            new VariableReferenceExpression(uv))));
+        }
+        OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
+        OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
+        context.computeAndSetTypeEnvironmentForOperator(g);
+        return true;
+    }
+
+    private NestedTupleSourceOperator getNts(AbstractLogicalOperator op) {
+        AbstractLogicalOperator alo = op;
+        do {
+            if (alo.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                return (NestedTupleSourceOperator) alo;
+            }
+            if (alo.getInputs().size() != 1) {
+                return null;
+            }
+            alo = (AbstractLogicalOperator) alo.getInputs().get(0).getValue();
+        } while (true);
+    }
+
+    protected Set<LogicalVariable> computeGbyVars(AbstractLogicalOperator op, Set<LogicalVariable> freeVars,
+            IOptimizationContext context) throws AlgebricksException {
+        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
+        List<FunctionalDependency> fdList = context.getFDList(op);
+        if (fdList == null) {
+            return null;
+        }
+        // check if any of the FDs is a key
+        List<LogicalVariable> all = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(op, all);
+        all.retainAll(freeVars);
+        for (FunctionalDependency fd : fdList) {
+            if (fd.getTail().containsAll(all)) {
+                return new HashSet<LogicalVariable>(fd.getHead());
+            }
+        }
+        return null;
+    }
+
+    private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars,
+            IOptimizationContext context, GroupByOperator g,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
+        Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>();
+        for (LogicalVariable ov : vars) {
+            LogicalVariable newVar = context.newVar();
+            ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
+            outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
+                    new MutableObject<ILogicalExpression>(varExpr)));
+            for (ILogicalPlan p : g.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
+                            context);
+                }
+            }
+            AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue();
+            OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context);
+            m.put(ov, newVar);
+        }
+        return m;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java
new file mode 100644
index 0000000..3c96bc2
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Projects away unused variables at the earliest possible point.
+ * Does a full DFS sweep of the plan adding ProjectOperators in the bottom-up pass.
+ * Also, removes projects that have become useless.
+ * TODO: This rule 'recklessly' adds as many projects as possible, but there is no guarantee
+ * that the overall cost of the plan is reduced since project operators also add a cost.
+ */
+public class IntroduceProjectsRule implements IAlgebraicRewriteRule {
+
+    private final Set<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
+    private final Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
+    private final Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
+    private final List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>();
+    protected boolean hasRun = false;
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (hasRun) {
+            return false;
+        }
+        hasRun = true;
+        return introduceProjects(null, -1, opRef, Collections.<LogicalVariable> emptySet(), context);
+    }
+
+    protected boolean introduceProjects(AbstractLogicalOperator parentOp, int parentInputIndex,
+            Mutable<ILogicalOperator> opRef, Set<LogicalVariable> parentUsedVars, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        boolean modified = false;
+
+        usedVars.clear();
+        VariableUtilities.getUsedVariables(op, usedVars);
+
+        // In the top-down pass, maintain a set of variables that are used in op and all its parents.
+        HashSet<LogicalVariable> parentsUsedVars = new HashSet<LogicalVariable>();
+        parentsUsedVars.addAll(parentUsedVars);
+        parentsUsedVars.addAll(usedVars);
+
+        // Descend into children.        
+        for (int i = 0; i < op.getInputs().size(); i++) {
+            Mutable<ILogicalOperator> inputOpRef = op.getInputs().get(i);
+            if (introduceProjects(op, i, inputOpRef, parentsUsedVars, context)) {
+                modified = true;
+            }
+        }
+
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(op);
+        }
+        // In the bottom-up pass, determine which live variables are not used by op's parents.
+        // Such variables are be projected away.
+        liveVars.clear();
+        VariableUtilities.getLiveVariables(op, liveVars);
+        producedVars.clear();
+        VariableUtilities.getProducedVariables(op, producedVars);
+        liveVars.removeAll(producedVars);
+
+        projectVars.clear();
+        for (LogicalVariable liveVar : liveVars) {
+            if (parentsUsedVars.contains(liveVar)) {
+                projectVars.add(liveVar);
+            }
+        }
+
+        // Some of the variables that are live at this op are not used above.
+        if (projectVars.size() != liveVars.size()) {
+            // Add a project operator under each of op's qualifying input branches.
+            for (int i = 0; i < op.getInputs().size(); i++) {
+                ILogicalOperator childOp = op.getInputs().get(i).getValue();
+                liveVars.clear();
+                VariableUtilities.getLiveVariables(childOp, liveVars);
+                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+                vars.addAll(projectVars);
+                // Only retain those variables that are live in the i-th input branch.
+                vars.retainAll(liveVars);
+                if (vars.size() != liveVars.size()) {
+                    ProjectOperator projectOp = new ProjectOperator(vars);
+                    projectOp.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
+                    op.getInputs().get(i).setValue(projectOp);
+                    context.computeAndSetTypeEnvironmentForOperator(projectOp);
+                    modified = true;
+                }
+            }
+        } else if (op.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+            // Check if the existing project has become useless.
+            liveVars.clear();
+            VariableUtilities.getLiveVariables(op.getInputs().get(0).getValue(), liveVars);
+            ProjectOperator projectOp = (ProjectOperator) op;
+            List<LogicalVariable> projectVars = projectOp.getVariables();
+            if (liveVars.size() == projectVars.size() && liveVars.containsAll(projectVars)) {
+                boolean eliminateProject = true;
+                // For UnionAll the variables must also be in exactly the correct order.
+                if (parentOp.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
+                    eliminateProject = canEliminateProjectBelowUnion((UnionAllOperator) parentOp, projectOp,
+                            parentInputIndex);
+                }
+                if (eliminateProject) {
+                    // The existing project has become useless. Remove it.
+                    parentOp.getInputs().get(parentInputIndex).setValue(op.getInputs().get(0).getValue());
+                }
+            }
+        }
+
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(op);
+        }
+        return modified;
+    }
+    
+    private boolean canEliminateProjectBelowUnion(UnionAllOperator unionOp, ProjectOperator projectOp,
+            int unionInputIndex) throws AlgebricksException {
+        List<LogicalVariable> orderedLiveVars = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(projectOp.getInputs().get(0).getValue(), orderedLiveVars);
+        int numVars = orderedLiveVars.size();
+        for (int i = 0; i < numVars; i++) {
+            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varTriple = unionOp.getVariableMappings().get(i);
+            if (unionInputIndex == 0) {
+                if (varTriple.first != orderedLiveVars.get(i)) {
+                    return false;
+                }
+            } else {
+                if (varTriple.second != orderedLiveVars.get(i)) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
new file mode 100644
index 0000000..71af64b
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IsolateHyracksOperatorsRule implements IAlgebraicRewriteRule {
+
+    private final PhysicalOperatorTag[] operatorsBelowWhichJobGenIsDisabled;
+
+    public IsolateHyracksOperatorsRule(PhysicalOperatorTag[] operatorsBelowWhichJobGenIsDisabled) {
+        this.operatorsBelowWhichJobGenIsDisabled = operatorsBelowWhichJobGenIsDisabled;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        IPhysicalOperator pt = op.getPhysicalOperator();
+
+        if (pt == null || op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+            return false;
+        }
+        if (!pt.isMicroOperator()) {
+            return testIfExchangeBelow(opRef, context);
+        } else {
+            return testIfExchangeAbove(opRef, context);
+        }
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    private boolean testIfExchangeBelow(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        boolean exchInserted = false;
+
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
+            if (c.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
+                if (c.getPhysicalOperator() == null) {
+                    return false;
+                }
+                insertOneToOneExchange(i, context);
+                exchInserted = true;
+            }
+        }
+        IPhysicalOperator pt = op.getPhysicalOperator();
+        if (pt.isJobGenDisabledBelowMe() || arrayContains(operatorsBelowWhichJobGenIsDisabled, pt.getOperatorTag())) {
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                disableJobGenRec(i.getValue());
+            }
+        }
+        return exchInserted;
+    }
+
+    private void disableJobGenRec(ILogicalOperator operator) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) operator;
+        op.disableJobGen();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            disableJobGenRec(i.getValue());
+        }
+    }
+
+    private boolean testIfExchangeAbove(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+            return false;
+        }
+        boolean exchInserted = false;
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
+            IPhysicalOperator cpop = c.getPhysicalOperator();
+            if (c.getOperatorTag() == LogicalOperatorTag.EXCHANGE || cpop == null) {
+                continue;
+            }
+            if (!cpop.isMicroOperator()) {
+                insertOneToOneExchange(i, context);
+                exchInserted = true;
+            }
+        }
+        return exchInserted;
+    }
+
+    private final static <T> boolean arrayContains(T[] array, T tag) {
+        for (int i = 0; i < array.length; i++) {
+            if (array[i] == tag) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private final static void insertOneToOneExchange(Mutable<ILogicalOperator> i, IOptimizationContext context)
+            throws AlgebricksException {
+        ExchangeOperator e = new ExchangeOperator();
+        e.setPhysicalOperator(new OneToOneExchangePOperator());
+        ILogicalOperator inOp = i.getValue();
+
+        e.getInputs().add(new MutableObject<ILogicalOperator>(inOp));
+        i.setValue(e);
+        // e.recomputeSchema();
+        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(e, context);
+        ExecutionMode em = ((AbstractLogicalOperator) inOp).getExecutionMode();
+        e.setExecutionMode(em);
+        e.computeDeliveredPhysicalProperties(context);
+        context.computeAndSetTypeEnvironmentForOperator(e);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java
new file mode 100644
index 0000000..5867abd
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.ListIterator;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.AbstractDecorrelationRule;
+
+/**
+ * The rule searches for operators that can be moved outside the subplan.
+ *
+ * <pre>
+ * Before
+ * 
+ *   %PARENT_PLAN
+ *   SUBPLAN{
+ *     %NESTED_OPERATORS_B+
+ *     ASSIGN || %SUBPLAN
+ *     %NESTED_OPERATORS_A*
+ *     NESTEDTUPLESOURCE
+ *   }
+ *   %CHILD_PLAN
+ * 
+ *   where
+ *     %SUBPLAN has one nested plan with a root AGGREGATE operator.
+ * 
+ * After
+ * 
+ *   %PARENT_PLAN
+ *   SUBPLAN{
+ *     %NESTED_OPERATORS_B+
+ *     %NESTED_OPERATORS_A*
+ *     NESTEDTUPLESOURCE
+ *   }
+ *   ASSIGN || %SUBPLAN
+ *   %CHILD_PLAN
+ * </pre>
+ */
+public class MoveFreeVariableOperatorOutOfSubplanRule extends AbstractDecorrelationRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op0;
+
+        Mutable<ILogicalOperator> leftRef = subplan.getInputs().get(0);
+        if (((AbstractLogicalOperator) leftRef.getValue()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+            return false;
+        }
+
+        ListIterator<ILogicalPlan> plansIter = subplan.getNestedPlans().listIterator();
+        ILogicalPlan p = null;
+        while (plansIter.hasNext()) {
+            p = plansIter.next();
+        }
+        if (p == null) {
+            return false;
+        }
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef1 = p.getRoots().get(0);
+
+        //The root operator will not be movable. Start with the second op
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getValue();
+        if (op1.getInputs().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+
+        //Get all variables that come from outside of the loop
+        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op1, free);
+
+        while (op2Ref != null) {
+            //Get the operator that we want to look at
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+
+            //Make sure we are looking at subplan with a scan/join
+            if (op2.getInputs().size() != 1 || !descOrSelfIsScanOrJoin(op2)) {
+                return false;
+            }
+            boolean notApplicable = false;
+
+            //Get its used variables
+            Set<LogicalVariable> used = new HashSet<LogicalVariable>();
+
+            //not movable if the operator is not an assign or subplan
+            //Might be helpful in the future for other operations in the future
+            if (movableOperator(op2.getOperatorTag())) {
+                if (op2.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+                    VariableUtilities.getUsedVariables(op2, used);
+                } else if (op2.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+                    // Nested plan must have an aggregate root.
+                    ListIterator<ILogicalPlan> subplansIter = ((SubplanOperator) op2).getNestedPlans().listIterator();
+                    ILogicalPlan plan = null;
+                    while (subplansIter.hasNext()) {
+                        plan = subplansIter.next();
+                    }
+                    if (plan == null) {
+                        return false;
+                    }
+                    if (plan.getRoots().size() != 1) {
+                        return false;
+                    }
+                    ILogicalOperator op3 = plan.getRoots().get(0).getValue();
+                    if (op3.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+                        return false;
+                    }
+                    // Used variables do not include ones created in the subplan.
+                    VariableUtilities.getUsedVariables(op2, used);
+                    Set<LogicalVariable> subplanProducedAndDown = new HashSet<LogicalVariable>();
+                    VariableUtilities.getProducedVariablesInDescendantsAndSelf(op3, subplanProducedAndDown);
+                    used.removeAll(subplanProducedAndDown);
+                } else {
+                    notApplicable = true;
+                }
+            } else {
+                notApplicable = true;
+            }
+
+            //Make sure that all of its used variables come from outside
+            for (LogicalVariable var : used) {
+                if (!free.contains(var)) {
+                    notApplicable = true;
+                }
+            }
+
+            if (notApplicable) {
+                op2Ref = op2.getInputs().get(0);
+            } else {
+                //Make the input of op2 be the input of op1
+                op2Ref.setValue(op2.getInputs().get(0).getValue());
+
+                //Make the outside of the subplan the input of op2
+                Mutable<ILogicalOperator> outsideRef = op2.getInputs().get(0);
+                outsideRef.setValue(op0.getInputs().get(0).getValue());
+
+                //Make op2 the input of the subplan
+                Mutable<ILogicalOperator> op2OutsideRef = op0.getInputs().get(0);
+                op2OutsideRef.setValue(op2);
+
+                return true;
+            }
+
+        }
+        return false;
+    }
+
+    protected boolean movableOperator(LogicalOperatorTag operatorTag) {
+        return (operatorTag == LogicalOperatorTag.ASSIGN || operatorTag == LogicalOperatorTag.SUBPLAN);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java
new file mode 100644
index 0000000..6484202
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * replace Subplan operators with nested loop joins where the join condition is true, if the Subplan
+ * does not contain free variables (does not have correlations to the input stream).
+ * 
+ * @author yingyib
+ */
+public class NestedSubplanToJoinRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue()))
+            return false;
+        context.addToDontApplySet(this, opRef.getValue());
+
+        ILogicalOperator op1 = opRef.getValue();
+        if (op1.getInputs().size() == 0) {
+            return false;
+        }
+
+        boolean rewritten = false;
+        for (int index = 0; index < op1.getInputs().size(); index++) {
+            AbstractLogicalOperator child = (AbstractLogicalOperator) op1.getInputs().get(index).getValue();
+            if (child.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+                continue;
+            }
+
+            AbstractOperatorWithNestedPlans subplan = (AbstractOperatorWithNestedPlans) child;
+            Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
+            OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
+            if (!freeVars.isEmpty()) {
+                /**
+                 * the subplan is correlated with the outer plan, other rules can deal with it
+                 */
+                continue;
+            }
+
+            /** get the input operator of the subplan operator */
+            ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
+            AbstractLogicalOperator subplanInputOp = (AbstractLogicalOperator) subplanInput;
+
+            /** If the other join branch is a trivial plan, do not do the rewriting. */
+            if (subplanInputOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+                continue;
+            }
+
+            /** get all nested top operators */
+            List<ILogicalPlan> nestedPlans = subplan.getNestedPlans();
+            List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<Mutable<ILogicalOperator>>();
+            for (ILogicalPlan nestedPlan : nestedPlans) {
+                nestedRoots.addAll(nestedPlan.getRoots());
+            }
+            if (nestedRoots.size() == 0) {
+                /** there is no nested top operators */
+                continue;
+            }
+
+            /** expend the input and roots into a DAG of nested loop joins */
+            Mutable<ILogicalExpression> expr = new MutableObject<ILogicalExpression>(ConstantExpression.TRUE);
+            Mutable<ILogicalOperator> nestedRootRef = nestedRoots.get(0);
+            ILogicalOperator join = new LeftOuterJoinOperator(expr, new MutableObject<ILogicalOperator>(subplanInput),
+                    nestedRootRef);
+
+            /** rewrite the nested tuple source to be empty tuple source */
+            rewriteNestedTupleSource(nestedRootRef);
+
+            for (int i = 1; i < nestedRoots.size(); i++) {
+                join = new LeftOuterJoinOperator(expr, new MutableObject<ILogicalOperator>(join), nestedRoots.get(i));
+            }
+            op1.getInputs().get(index).setValue(join);
+            context.computeAndSetTypeEnvironmentForOperator(join);
+            rewritten = true;
+        }
+        return rewritten;
+    }
+
+    /**
+     * rewrite NestedTupleSource operators to EmptyTupleSource operators
+     * 
+     * @param nestedRootRef
+     */
+    private void rewriteNestedTupleSource(Mutable<ILogicalOperator> nestedRootRef) {
+        AbstractLogicalOperator nestedRoot = (AbstractLogicalOperator) nestedRootRef.getValue();
+        if (nestedRoot.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            nestedRootRef.setValue(new EmptyTupleSourceOperator());
+        }
+        List<Mutable<ILogicalOperator>> inputs = nestedRoot.getInputs();
+        for (Mutable<ILogicalOperator> input : inputs) {
+            rewriteNestedTupleSource(input);
+        }
+    }
+}

[17/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
new file mode 100644
index 0000000..6ab624d
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class PigletFileDataSink implements IDataSink {
+    private String file;
+
+    private FileSplit[] fileSplits;
+
+    private IPartitioningProperty partProp;
+
+    public PigletFileDataSink(String file) {
+        this.file = file;
+        fileSplits = FileSplitUtils.parseFileSplits(file);
+        partProp = new RandomPartitioningProperty(new FileSplitDomain(fileSplits));
+    }
+
+    @Override
+    public Object getId() {
+        return file;
+    }
+
+    public FileSplit[] getFileSplits() {
+        return fileSplits;
+    }
+
+    @Override
+    public Object[] getSchemaTypes() {
+        return null;
+    }
+
+    @Override
+    public IPartitioningProperty getPartitioningProperty() {
+        return partProp;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
new file mode 100644
index 0000000..076e15a
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class PigletFileDataSource implements IDataSource<String> {
+    private final String file;
+
+    private final Object[] types;
+
+    private final FileSplit[] fileSplits;
+
+    private IDataSourcePropertiesProvider propProvider;
+
+    public PigletFileDataSource(String file, Object[] types) {
+        this.file = file;
+        this.types = types;
+        fileSplits = FileSplitUtils.parseFileSplits(file);
+        final IPhysicalPropertiesVector vec = new StructuralPropertiesVector(new RandomPartitioningProperty(
+                new FileSplitDomain(fileSplits)), new ArrayList<ILocalStructuralProperty>());
+        propProvider = new IDataSourcePropertiesProvider() {
+            @Override
+            public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
+                return vec;
+            }
+        };
+    }
+
+    @Override
+    public String getId() {
+        return file;
+    }
+
+    @Override
+    public Object[] getSchemaTypes() {
+        return types;
+    }
+
+    public FileSplit[] getFileSplits() {
+        return fileSplits;
+    }
+
+    @Override
+    public IDataSourcePropertiesProvider getPropertiesProvider() {
+        return propProvider;
+    }
+
+    @Override
+    public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList) {
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
new file mode 100644
index 0000000..ba5c5aa
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AbstractFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+
+public class PigletFunction extends AbstractFunctionInfo {
+    private static final long serialVersionUID = 1L;
+
+    private final FunctionIdentifier fid;
+
+    public PigletFunction(FunctionIdentifier fid) {
+        super(true);
+        this.fid = fid;
+    }
+
+    @Override
+    public FunctionIdentifier getFunctionIdentifier() {
+        return fid;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
new file mode 100644
index 0000000..099520d
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
@@ -0,0 +1,234 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.operators.std.SinkWriterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+
+public class PigletMetadataProvider implements IMetadataProvider<String, String> {
+    private static final Map<FunctionIdentifier, PigletFunction> FN_MAP;
+
+    static {
+        Map<FunctionIdentifier, PigletFunction> map = new HashMap<FunctionIdentifier, PigletFunction>();
+
+        map.put(AlgebricksBuiltinFunctions.EQ, new PigletFunction(AlgebricksBuiltinFunctions.EQ));
+
+        FN_MAP = Collections.unmodifiableMap(map);
+    }
+
+    @Override
+    public IDataSource<String> findDataSource(String id) throws AlgebricksException {
+        return null;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource,
+            List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+            List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+            IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+            throws AlgebricksException {
+        PigletFileDataSource ds = (PigletFileDataSource) dataSource;
+
+        FileSplit[] fileSplits = ds.getFileSplits();
+        String[] locations = new String[fileSplits.length];
+        for (int i = 0; i < fileSplits.length; ++i) {
+            locations[i] = fileSplits[i].getNodeName();
+        }
+        IFileSplitProvider fsp = new ConstantFileSplitProvider(fileSplits);
+
+        Object[] colTypes = ds.getSchemaTypes();
+        IValueParserFactory[] vpfs = new IValueParserFactory[colTypes.length];
+        ISerializerDeserializer[] serDesers = new ISerializerDeserializer[colTypes.length];
+
+        for (int i = 0; i < colTypes.length; ++i) {
+            Type colType = (Type) colTypes[i];
+            IValueParserFactory vpf;
+            ISerializerDeserializer serDeser;
+            switch (colType.getTag()) {
+                case INTEGER:
+                    vpf = IntegerParserFactory.INSTANCE;
+                    serDeser = IntegerSerializerDeserializer.INSTANCE;
+                    break;
+
+                case CHAR_ARRAY:
+                    vpf = UTF8StringParserFactory.INSTANCE;
+                    serDeser = UTF8StringSerializerDeserializer.INSTANCE;
+                    break;
+
+                case FLOAT:
+                    vpf = FloatParserFactory.INSTANCE;
+                    serDeser = FloatSerializerDeserializer.INSTANCE;
+                    break;
+
+                default:
+                    throw new UnsupportedOperationException();
+            }
+            vpfs[i] = vpf;
+            serDesers[i] = serDeser;
+        }
+
+        ITupleParserFactory tpf = new DelimitedDataTupleParserFactory(vpfs, ',');
+        RecordDescriptor rDesc = new RecordDescriptor(serDesers);
+
+        IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
+        AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
+        return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
+    }
+
+    @Override
+    public boolean scannerOperatorIsLeaf(IDataSource<String> dataSource) {
+        return true;
+    }
+
+    @Override
+    public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
+            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
+            throws AlgebricksException {
+        PigletFileDataSink ds = (PigletFileDataSink) sink;
+        FileSplit[] fileSplits = ds.getFileSplits();
+        String[] locations = new String[fileSplits.length];
+        for (int i = 0; i < fileSplits.length; ++i) {
+            locations[i] = fileSplits[i].getNodeName();
+        }
+        IPushRuntimeFactory prf = new SinkWriterRuntimeFactory(printColumns, printerFactories, fileSplits[0]
+                .getLocalFile().getFile(), PrinterBasedWriterFactory.INSTANCE, inputDesc);
+        AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
+        return new Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint>(prf, constraint);
+    }
+
+    @Override
+    public IDataSourceIndex<String, String> findDataSourceIndex(String indexId, String dataSourceId)
+            throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
+            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
+            JobSpecification spec) throws AlgebricksException {
+        return null;
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(
+            IDataSource<String> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> keys,
+            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, JobGenContext context,
+            JobSpecification jobSpec) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
+            IDataSourceIndex<String, String> dataSource,
+            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+            IVariableTypeEnvironment typeEnv,
+            List<LogicalVariable> primaryKeys,
+            List<LogicalVariable> secondaryKeys,
+            List<LogicalVariable> additionalNonKeyFields,
+            ILogicalExpression filterExpr,
+            RecordDescriptor recordDesc, JobGenContext context,
+            JobSpecification spec, boolean bulkload) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
+            IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
+            IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
+            List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
+            ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
+            throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(
+            IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
+            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+            ILogicalExpression filterExpr, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public IFunctionInfo lookupFunction(FunctionIdentifier fid) {
+        return FN_MAP.get(fid);
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
+            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification jobSpec, boolean bulkload) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+    @Override
+    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<String> dataSource,
+            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
+            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
+            JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+        // TODO Auto-generated method stub
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
new file mode 100644
index 0000000..1c2cfae
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.rewriter;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.BreakSelectIntoConjunctsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ComplexJoinInferenceRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ConsolidateAssignsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ConsolidateSelectsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.EliminateSubplanRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractGbyExpressionsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecorVarsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.InferTypesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.InlineVariablesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.IsolateHyracksOperatorsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PullSelectOutOfEqJoin;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.CopyLimitDownRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushProjectDownRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushProjectIntoDataSourceScanRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushSelectDownRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushSelectIntoJoinRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ReinferAllTypesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.RemoveUnusedAssignAndAggregateRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetAlgebricksPhysicalOperatorsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetExecutionModeRule;
+
+public class PigletRewriteRuleset {
+
+    public final static List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() {
+        List<IAlgebraicRewriteRule> typeInfer = new LinkedList<IAlgebraicRewriteRule>();
+        typeInfer.add(new InferTypesRule());
+        return typeInfer;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() {
+        List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>();
+        normalization.add(new EliminateSubplanRule());
+        // TODO: This rule is incorrect and has been removed. Its replacement in
+        // Asterix (PushAggFuncIntoStandaloneAggregateRule)
+        // is language-specific.
+        // normalization.add(new IntroduceGroupByForStandaloneAggregRule());
+        normalization.add(new BreakSelectIntoConjunctsRule());
+        normalization.add(new PushSelectIntoJoinRule());
+        normalization.add(new ExtractGbyExpressionsRule());
+        return normalization;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildCondPushDownRuleCollection() {
+        List<IAlgebraicRewriteRule> condPushDown = new LinkedList<IAlgebraicRewriteRule>();
+        condPushDown.add(new PushSelectDownRule());
+        condPushDown.add(new InlineVariablesRule());
+        condPushDown.add(new FactorRedundantGroupAndDecorVarsRule());
+        condPushDown.add(new EliminateSubplanRule());
+        return condPushDown;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildJoinInferenceRuleCollection() {
+        List<IAlgebraicRewriteRule> joinInference = new LinkedList<IAlgebraicRewriteRule>();
+        joinInference.add(new InlineVariablesRule());
+        joinInference.add(new ComplexJoinInferenceRule());
+        return joinInference;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildOpPushDownRuleCollection() {
+        List<IAlgebraicRewriteRule> opPushDown = new LinkedList<IAlgebraicRewriteRule>();
+        opPushDown.add(new PushProjectDownRule());
+        opPushDown.add(new PushSelectDownRule());
+        return opPushDown;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() {
+        List<IAlgebraicRewriteRule> dataExchange = new LinkedList<IAlgebraicRewriteRule>();
+        dataExchange.add(new SetExecutionModeRule());
+        return dataExchange;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() {
+        List<IAlgebraicRewriteRule> consolidation = new LinkedList<IAlgebraicRewriteRule>();
+        consolidation.add(new ConsolidateSelectsRule());
+        consolidation.add(new ConsolidateAssignsRule());
+        consolidation.add(new RemoveUnusedAssignAndAggregateRule());
+        return consolidation;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() {
+        List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>();
+        physicalPlanRewrites.add(new PullSelectOutOfEqJoin());
+        physicalPlanRewrites.add(new SetAlgebricksPhysicalOperatorsRule());
+        physicalPlanRewrites.add(new EnforceStructuralPropertiesRule());
+        physicalPlanRewrites.add(new PushProjectDownRule());
+        physicalPlanRewrites.add(new CopyLimitDownRule());
+        return physicalPlanRewrites;
+    }
+
+    public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection() {
+        List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>();
+        physicalPlanRewrites.add(new CopyLimitDownRule());
+        return physicalPlanRewrites;
+    }
+
+    public final static List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() {
+        List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<IAlgebraicRewriteRule>();
+        prepareForJobGenRewrites.add(new IsolateHyracksOperatorsRule(
+                HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
+        prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule());
+        // Re-infer all types, so that, e.g., the effect of not-is-null is
+        // propagated.
+        prepareForJobGenRewrites.add(new PushProjectIntoDataSourceScanRule());
+        prepareForJobGenRewrites.add(new ReinferAllTypesRule());
+        return prepareForJobGenRewrites;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
new file mode 100644
index 0000000..9ae68bb
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime;
+
+import java.io.DataOutput;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.ConstantValue;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions.PigletFunctionRegistry;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
+import edu.uci.ics.hyracks.algebricks.runtime.evaluators.ConstantEvalFactory;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+
+public class PigletExpressionJobGen implements ILogicalExpressionJobGen {
+    @Override
+    public ICopyEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
+        switch (expr.getExpressionTag()) {
+            case CONSTANT: {
+                ConstantValue cv = (ConstantValue) ((ConstantExpression) expr).getValue();
+                Type type = cv.getType();
+                String image = cv.getImage();
+                ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+                DataOutput dos = abvs.getDataOutput();
+                switch (type.getTag()) {
+                    case INTEGER:
+                        try {
+                            IntegerSerializerDeserializer.INSTANCE.serialize(Integer.valueOf(image), dos);
+                        } catch (Exception e) {
+                            throw new AlgebricksException(e);
+                        }
+                        break;
+
+                    case CHAR_ARRAY:
+                        try {
+                            UTF8StringSerializerDeserializer.INSTANCE.serialize(image, dos);
+                        } catch (Exception e) {
+                            throw new AlgebricksException(e);
+                        }
+                        break;
+
+                    default:
+                        throw new UnsupportedOperationException("Unsupported constant type: " + type.getTag());
+                }
+                return new ConstantEvalFactory(Arrays.copyOf(abvs.getByteArray(), abvs.getLength()));
+            }
+
+            case FUNCTION_CALL: {
+                ScalarFunctionCallExpression sfce = (ScalarFunctionCallExpression) expr;
+
+                List<Mutable<ILogicalExpression>> argExprs = sfce.getArguments();
+                ICopyEvaluatorFactory argEvalFactories[] = new ICopyEvaluatorFactory[argExprs.size()];
+                for (int i = 0; i < argEvalFactories.length; ++i) {
+                    Mutable<ILogicalExpression> er = argExprs.get(i);
+                    argEvalFactories[i] = createEvaluatorFactory(er.getValue(), env, inputSchemas, context);
+                }
+                ICopyEvaluatorFactory funcEvalFactory;
+                try {
+                    funcEvalFactory = PigletFunctionRegistry.createFunctionEvaluatorFactory(
+                            sfce.getFunctionIdentifier(), argEvalFactories);
+                } catch (PigletException e) {
+                    throw new AlgebricksException(e);
+                }
+                return funcEvalFactory;
+            }
+
+            case VARIABLE: {
+                LogicalVariable var = ((VariableReferenceExpression) expr).getVariableReference();
+                int index = inputSchemas[0].findVariable(var);
+                return new ColumnAccessEvalFactory(index);
+            }
+        }
+        throw new IllegalArgumentException("Unknown expression type: " + expr.getExpressionTag());
+    }
+
+    @Override
+    public ICopyAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+            JobGenContext context) throws AlgebricksException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ICopyRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public ICopyUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+            throws AlgebricksException {
+        throw new UnsupportedOperationException();
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
new file mode 100644
index 0000000..c52a4bd
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+
+public interface IPigletFunctionEvaluatorFactoryBuilder {
+    public ICopyEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] arguments);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
new file mode 100644
index 0000000..e84964d
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerEqFunctionEvaluatorFactory implements ICopyEvaluatorFactory {
+    private static final long serialVersionUID = 1L;
+
+    private final ICopyEvaluatorFactory arg1Factory;
+
+    private final ICopyEvaluatorFactory arg2Factory;
+
+    public IntegerEqFunctionEvaluatorFactory(ICopyEvaluatorFactory arg1Factory, ICopyEvaluatorFactory arg2Factory) {
+        this.arg1Factory = arg1Factory;
+        this.arg2Factory = arg2Factory;
+    }
+
+    @Override
+    public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+        return new ICopyEvaluator() {
+            private DataOutput dataout = output.getDataOutput();
+            private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
+            private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
+            private ICopyEvaluator eval1 = arg1Factory.createEvaluator(out1);
+            private ICopyEvaluator eval2 = arg2Factory.createEvaluator(out2);
+
+            @Override
+            public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+                out1.reset();
+                eval1.evaluate(tuple);
+                out2.reset();
+                eval2.evaluate(tuple);
+                int v1 = IntegerPointable.getInteger(out1.getByteArray(), 0);
+                int v2 = IntegerPointable.getInteger(out2.getByteArray(), 0);
+                boolean r = v1 == v2;
+                try {
+                    dataout.writeBoolean(r);
+                } catch (IOException ioe) {
+                    throw new AlgebricksException(ioe);
+                }
+            }
+        };
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
new file mode 100644
index 0000000..35015a5
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+
+public class PigletFunctionRegistry {
+    private static final Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> builderMap;
+
+    static {
+        Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> temp = new HashMap<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder>();
+
+        temp.put(AlgebricksBuiltinFunctions.EQ, new IPigletFunctionEvaluatorFactoryBuilder() {
+            @Override
+            public ICopyEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] arguments) {
+                return new IntegerEqFunctionEvaluatorFactory(arguments[0], arguments[1]);
+            }
+        });
+
+        builderMap = Collections.unmodifiableMap(temp);
+    }
+
+    public static ICopyEvaluatorFactory createFunctionEvaluatorFactory(FunctionIdentifier fid, ICopyEvaluatorFactory[] args)
+            throws PigletException {
+        IPigletFunctionEvaluatorFactoryBuilder builder = builderMap.get(fid);
+        if (builder == null) {
+            throw new PigletException("Unknown function: " + fid);
+        }
+        return builder.buildEvaluatorFactory(fid, args);
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/BagType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/BagType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/BagType.java
new file mode 100644
index 0000000..9aec8bb
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/BagType.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class BagType extends Type {
+    @Override
+    public Tag getTag() {
+        return Tag.BAG;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/CharArrayType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/CharArrayType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/CharArrayType.java
new file mode 100644
index 0000000..4bef8ff
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/CharArrayType.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class CharArrayType extends Type {
+    public static final Type INSTANCE = new CharArrayType();
+
+    private CharArrayType() {
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.CHAR_ARRAY;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/DoubleType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/DoubleType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/DoubleType.java
new file mode 100644
index 0000000..8b7a0bd
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/DoubleType.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class DoubleType extends Type {
+    public static final Type INSTANCE = new DoubleType();
+
+    private DoubleType() {
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.DOUBLE;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/FloatType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/FloatType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/FloatType.java
new file mode 100644
index 0000000..69f5db6
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/FloatType.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class FloatType extends Type {
+    public static final Type INSTANCE = new FloatType();
+
+    private FloatType() {
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.FLOAT;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/IntegerType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/IntegerType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/IntegerType.java
new file mode 100644
index 0000000..9c568df
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/IntegerType.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class IntegerType extends Type {
+    public static final Type INSTANCE = new IntegerType();
+
+    private IntegerType() {
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.INTEGER;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/LongType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/LongType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/LongType.java
new file mode 100644
index 0000000..b992afc
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/LongType.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class LongType extends Type {
+    public static final Type INSTANCE = new LongType();
+
+    private LongType() {
+    }
+
+    @Override
+    public Tag getTag() {
+        return Tag.LONG;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/MapType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/MapType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/MapType.java
new file mode 100644
index 0000000..a46e40d
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/MapType.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class MapType extends Type {
+    @Override
+    public Tag getTag() {
+        return Tag.MAP;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Schema.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Schema.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Schema.java
new file mode 100644
index 0000000..1702353
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Schema.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+
+public class Schema {
+    private List<Pair<String, Type>> schema;
+
+    public Schema(List<Pair<String, Type>> schema) {
+        this.schema = schema;
+    }
+
+    public List<Pair<String, Type>> getSchema() {
+        return schema;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/TupleType.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/TupleType.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/TupleType.java
new file mode 100644
index 0000000..429026f
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/TupleType.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class TupleType extends Type {
+    @Override
+    public Tag getTag() {
+        return Tag.TUPLE;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Type.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Type.java b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Type.java
new file mode 100644
index 0000000..36efcea
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/main/java/org/apache/hyracks/algebricks/examples/piglet/types/Type.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public abstract class Type {
+    public enum Tag {
+        INTEGER,
+        LONG,
+        FLOAT,
+        DOUBLE,
+        CHAR_ARRAY,
+        TUPLE,
+        BAG,
+        MAP
+    }
+    
+    public abstract Tag getTag();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java b/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java
deleted file mode 100644
index e50b442..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.algebricks.examples.piglet.test;
-
-import java.io.File;
-
-import junit.framework.Test;
-import junit.framework.TestSuite;
-
-public class PigletTest {
-    public static Test suite() {
-        TestSuite suite = new TestSuite();
-        File dir = new File("testcases");
-        findAndAddTests(suite, dir);
-
-        return suite;
-    }
-
-    private static void findAndAddTests(TestSuite suite, File dir) {
-        for (final File f : dir.listFiles()) {
-            if (f.getName().startsWith(".")) {
-                continue;
-            }
-            if (f.isDirectory()) {
-                findAndAddTests(suite, f);
-            } else if (f.getName().endsWith(".piglet")) {
-                suite.addTest(new PigletTestCase(f));
-            }
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java b/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java
deleted file mode 100644
index 082a0b0..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.algebricks.examples.piglet.test;
-
-import java.io.File;
-import java.io.FileReader;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.PigletCompiler;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class PigletTestCase extends TestCase {
-
-    private final File file;
-
-    PigletTestCase(File file) {
-        super("testPiglet");
-        this.file = file;
-    }
-
-    @Test
-    public void testPiglet() {
-        try {
-            FileReader in = new FileReader(file);
-            try {
-                PigletCompiler c = new PigletCompiler();
-
-                List<ASTNode> ast = c.parse(in);
-                JobSpecification jobSpec = c.compile(ast);
-
-                System.err.println(jobSpec.toJSON());
-            } finally {
-                in.close();
-            }
-        } catch (Exception e) {
-            throw new RuntimeException(e);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTest.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTest.java b/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTest.java
new file mode 100644
index 0000000..e50b442
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTest.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.algebricks.examples.piglet.test;
+
+import java.io.File;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+public class PigletTest {
+    public static Test suite() {
+        TestSuite suite = new TestSuite();
+        File dir = new File("testcases");
+        findAndAddTests(suite, dir);
+
+        return suite;
+    }
+
+    private static void findAndAddTests(TestSuite suite, File dir) {
+        for (final File f : dir.listFiles()) {
+            if (f.getName().startsWith(".")) {
+                continue;
+            }
+            if (f.isDirectory()) {
+                findAndAddTests(suite, f);
+            } else if (f.getName().endsWith(".piglet")) {
+                suite.addTest(new PigletTestCase(f));
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTestCase.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTestCase.java b/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTestCase.java
new file mode 100644
index 0000000..082a0b0
--- /dev/null
+++ b/algebricks/algebricks-examples/piglet-example/src/test/java/org/apache/algebricks/examples/piglet/test/PigletTestCase.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.algebricks.examples.piglet.test;
+
+import java.io.File;
+import java.io.FileReader;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.PigletCompiler;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class PigletTestCase extends TestCase {
+
+    private final File file;
+
+    PigletTestCase(File file) {
+        super("testPiglet");
+        this.file = file;
+    }
+
+    @Test
+    public void testPiglet() {
+        try {
+            FileReader in = new FileReader(file);
+            try {
+                PigletCompiler c = new PigletCompiler();
+
+                List<ASTNode> ast = c.parse(in);
+                JobSpecification jobSpec = c.compile(ast);
+
+                System.err.println(jobSpec.toJSON());
+            } finally {
+                in.close();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
deleted file mode 100644
index f897791..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
-
-public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    protected boolean descOrSelfIsScanOrJoin(AbstractLogicalOperator op2) {
-        LogicalOperatorTag t = op2.getOperatorTag();
-        if (isScanOrJoin(t)) {
-            return true;
-        }
-        if (op2.getInputs().size() != 1) {
-            return false;
-        }
-        AbstractLogicalOperator alo = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
-        if (descOrSelfIsScanOrJoin(alo)) {
-            return true;
-        }
-        return false;
-    }
-
-    protected boolean isScanOrJoin(LogicalOperatorTag t) {
-        if (t == LogicalOperatorTag.DATASOURCESCAN || t == LogicalOperatorTag.INNERJOIN
-                || t == LogicalOperatorTag.UNNEST || t == LogicalOperatorTag.UNNEST_MAP
-                || t == LogicalOperatorTag.LEFTOUTERJOIN) {
-            return true;
-        }
-        return false;
-    }
-
-    protected Set<LogicalVariable> computeGbyVarsUsingPksOnly(Set<LogicalVariable> varSet, AbstractLogicalOperator op,
-            IOptimizationContext context) throws AlgebricksException {
-        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
-        List<FunctionalDependency> fdList = context.getFDList(op);
-        if (fdList == null) {
-            return null;
-        }
-        // check if any of the FDs is a key
-        for (FunctionalDependency fd : fdList) {
-            if (fd.getTail().containsAll(varSet)) {
-                return new HashSet<LogicalVariable>(fd.getHead());
-            }
-        }
-        return null;
-    }
-
-    protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g,
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
-        for (LogicalVariable ov : vars) {
-            LogicalVariable newVar = context.newVar();
-            ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
-            outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
-                    new MutableObject<ILogicalExpression>(varExpr)));
-            for (ILogicalPlan p : g.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
-                            context);
-                }
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
deleted file mode 100644
index 882db41..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public abstract class AbstractExtractExprRule implements IAlgebraicRewriteRule {
-
-    protected LogicalVariable extractExprIntoAssignOpRef(ILogicalExpression gExpr, Mutable<ILogicalOperator> opRef2,
-            IOptimizationContext context) throws AlgebricksException {
-        LogicalVariable v = context.newVar();
-        AssignOperator a = new AssignOperator(v, new MutableObject<ILogicalExpression>(gExpr));
-        a.getInputs().add(new MutableObject<ILogicalOperator>(opRef2.getValue()));
-        opRef2.setValue(a);
-        if (gExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
-            context.addNotToBeInlinedVar(v);
-        }
-        context.computeAndSetTypeEnvironmentForOperator(a);
-        return v;
-    }
-
-}

[37/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
deleted file mode 100644
index c267364..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public class LocalGroupingProperty extends AbstractGroupingProperty implements ILocalStructuralProperty {
-
-    // preferredOrderEnforcer, if not null, is guaranteed to enforce grouping on
-    // columnSet
-    private List<LogicalVariable> preferredOrderEnforcer;
-
-    public LocalGroupingProperty(Set<LogicalVariable> columnSet) {
-        super(columnSet);
-    }
-
-    public LocalGroupingProperty(Set<LogicalVariable> columnSet, List<LogicalVariable> preferredOrderEnforcer) {
-        this(columnSet);
-        this.preferredOrderEnforcer = preferredOrderEnforcer;
-    }
-
-    @Override
-    public PropertyType getPropertyType() {
-        return PropertyType.LOCAL_GROUPING_PROPERTY;
-    }
-
-    @Override
-    public void getColumns(Collection<LogicalVariable> columns) {
-        columns.addAll(columnSet);
-    }
-
-    @Override
-    public String toString() {
-        return columnSet.toString();
-    }
-
-    @Override
-    public void getVariables(Collection<LogicalVariable> variables) {
-        variables.addAll(columnSet);
-    }
-
-    public List<LogicalVariable> getPreferredOrderEnforcer() {
-        return preferredOrderEnforcer;
-    }
-
-    @Override
-    public ILocalStructuralProperty retainVariables(Collection<LogicalVariable> vars) {
-        Set<LogicalVariable> newVars = new ListSet<LogicalVariable>();
-        newVars.addAll(vars);
-        newVars.retainAll(columnSet);
-        if (columnSet.equals(newVars)) {
-            return new LocalGroupingProperty(columnSet, preferredOrderEnforcer);
-        }
-        // Column set for the retained grouping property
-        Set<LogicalVariable> newColumns = new ListSet<LogicalVariable>();
-        // Matches the prefix of the original column set.
-        for (LogicalVariable v : columnSet) {
-            if (newVars.contains(v)) {
-                newColumns.add(v);
-            } else {
-                break;
-            }
-        }
-        if (newColumns.size() > 0) {
-            return new LocalGroupingProperty(newColumns, preferredOrderEnforcer.subList(0, newColumns.size()));
-        } else {
-            return null;
-        }
-    }
-
-    @Override
-    public ILocalStructuralProperty regardToGroup(Collection<LogicalVariable> groupKeys) {
-        Set<LogicalVariable> newColumns = new ListSet<LogicalVariable>();
-        for (LogicalVariable v : columnSet) {
-            if (!groupKeys.contains(v)) {
-                newColumns.add(v);
-            }
-        }
-        if (newColumns.size() > 0) {
-            return new LocalGroupingProperty(newColumns, preferredOrderEnforcer.subList(groupKeys.size(),
-                    newColumns.size()));
-        } else {
-            return null;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
deleted file mode 100644
index 6d568f4..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-
-public final class LocalOrderProperty implements ILocalStructuralProperty {
-
-    private List<OrderColumn> orderColumns;
-
-    public LocalOrderProperty(List<OrderColumn> orderColumn) {
-        this.orderColumns = orderColumn;
-    }
-
-    public List<OrderColumn> getOrderColumns() {
-        return orderColumns;
-    }
-
-    public void setOrderColumns(List<OrderColumn> orderColumn) {
-        this.orderColumns = orderColumn;
-    }
-
-    public List<LogicalVariable> getColumns() {
-        List<LogicalVariable> orderVars = new ArrayList<LogicalVariable>();
-        for (OrderColumn oc : orderColumns) {
-            orderVars.add(oc.getColumn());
-        }
-        return orderVars;
-    }
-
-    public List<OrderKind> getOrders() {
-        List<OrderKind> orderKinds = new ArrayList<OrderKind>();
-        for (OrderColumn oc : orderColumns) {
-            orderKinds.add(oc.getOrder());
-        }
-        return orderKinds;
-    }
-
-    @Override
-    public PropertyType getPropertyType() {
-        return PropertyType.LOCAL_ORDER_PROPERTY;
-    }
-
-    @Override
-    public void getColumns(Collection<LogicalVariable> columns) {
-        columns.addAll(getColumns());
-    }
-
-    @Override
-    public String toString() {
-        return orderColumns.toString();
-    }
-
-    @Override
-    public void getVariables(Collection<LogicalVariable> variables) {
-        variables.addAll(getColumns());
-    }
-
-    @Override
-    public boolean equals(Object object) {
-        LocalOrderProperty lop = (LocalOrderProperty) object;
-        return orderColumns.equals(lop.orderColumns);
-    }
-
-    /**
-     * Whether current property implies the required property
-     * 
-     * @param required
-     *            , a required property
-     * @return true if the current property satisfies the required property; otherwise false.
-     */
-    public final boolean implies(ILocalStructuralProperty required) {
-        if (required.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
-            return false;
-        }
-        LocalOrderProperty requiredOrderProperty = (LocalOrderProperty) required;
-        Iterator<OrderColumn> requiredColumnIterator = requiredOrderProperty.getOrderColumns().iterator();
-        Iterator<OrderColumn> currentColumnIterator = orderColumns.iterator();
-
-        // Returns true if requiredColumnIterator is a prefix of currentColumnIterator.
-        return isPrefixOf(requiredColumnIterator, currentColumnIterator);
-    }
-
-    private <T> boolean isPrefixOf(Iterator<T> requiredColumnIterator, Iterator<T> currentColumnIterator) {
-        while (requiredColumnIterator.hasNext()) {
-            T oc = requiredColumnIterator.next();
-            if (!currentColumnIterator.hasNext()) {
-                return false;
-            }
-            if (!oc.equals(currentColumnIterator.next())) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    public final void normalizeOrderingColumns(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
-            List<FunctionalDependency> fds) {
-        replaceOrderingColumnsByEqClasses(equivalenceClasses);
-        applyFDsToOrderingColumns(fds);
-    }
-
-    private void replaceOrderingColumnsByEqClasses(Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
-        if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
-            return;
-        }
-        List<OrderColumn> norm = new ArrayList<OrderColumn>();
-        for (OrderColumn oc : orderColumns) {
-            LogicalVariable v = oc.getColumn();
-            EquivalenceClass ec = equivalenceClasses.get(v);
-            if (ec == null) {
-                norm.add(new OrderColumn(v, oc.getOrder()));
-            } else {
-                if (ec.representativeIsConst()) {
-                    // trivially satisfied, so the var. can be removed
-                } else {
-                    norm.add(new OrderColumn(ec.getVariableRepresentative(), oc.getOrder()));
-                }
-            }
-        }
-        orderColumns = norm;
-    }
-
-    private void applyFDsToOrderingColumns(List<FunctionalDependency> fds) {
-        if (fds == null || fds.isEmpty()) {
-            return;
-        }
-        Set<LogicalVariable> norm = new ListSet<LogicalVariable>();
-        List<LogicalVariable> columns = getColumns();
-        for (LogicalVariable v : columns) {
-            boolean isImpliedByAnFD = false;
-            for (FunctionalDependency fdep : fds) {
-                if (columns.containsAll(fdep.getHead()) && fdep.getTail().contains(v)) {
-                    isImpliedByAnFD = true;
-                    norm.addAll(fdep.getHead());
-                    break;
-                }
-
-            }
-            if (!isImpliedByAnFD) {
-                norm.add(v);
-            }
-        }
-        Set<OrderColumn> impliedColumns = new ListSet<OrderColumn>();
-        for (OrderColumn oc : orderColumns) {
-            if (!norm.contains(oc.getColumn())) {
-                impliedColumns.add(oc);
-            }
-        }
-        orderColumns.removeAll(impliedColumns);
-    }
-
-    @Override
-    public ILocalStructuralProperty retainVariables(Collection<LogicalVariable> vars) {
-        List<LogicalVariable> columns = getColumns();
-        List<LogicalVariable> newVars = new ArrayList<LogicalVariable>();
-        newVars.addAll(vars);
-        newVars.retainAll(columns);
-        List<OrderColumn> newColumns = new ArrayList<OrderColumn>();
-        for (OrderColumn oc : orderColumns) {
-            if (newVars.contains(oc.getColumn())) {
-                newColumns.add(oc);
-            } else {
-                break;
-            }
-        }
-        if (newColumns.size() > 0) {
-            return new LocalOrderProperty(newColumns);
-        } else {
-            return null;
-        }
-    }
-
-    @Override
-    public ILocalStructuralProperty regardToGroup(Collection<LogicalVariable> groupKeys) {
-        List<OrderColumn> newColumns = new ArrayList<OrderColumn>();
-        for (OrderColumn oc : orderColumns) {
-            if (!groupKeys.contains(oc.getColumn())) {
-                newColumns.add(oc);
-            }
-        }
-        if (newColumns.size() > 0) {
-            return new LocalOrderProperty(newColumns);
-        } else {
-            return null;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
deleted file mode 100644
index 848c86e..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-public class LogicalPropertiesVectorImpl implements ILogicalPropertiesVector {
-
-    private Integer numTuples, maxOutputFrames;
-
-    @Override
-    public String toString() {
-        return "LogicalPropertiesVector [ num.tuples: " + numTuples + ", maxOutputFrames: " + maxOutputFrames + " ]";
-    }
-
-    @Override
-    public Integer getNumberOfTuples() {
-        return numTuples;
-    }
-
-    public void setNumberOfTuples(Integer numTuples) {
-        this.numTuples = numTuples;
-    }
-
-    @Override
-    public Integer getMaxOutputFrames() {
-        return maxOutputFrames;
-    }
-
-    public void setMaxOutputFrames(Integer maxOutputFrames) {
-        this.maxOutputFrames = maxOutputFrames;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java
deleted file mode 100644
index a0e76d1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
-
-public final class OrderColumn {
-
-    private LogicalVariable column;
-    private OrderKind order;
-
-    public OrderColumn(LogicalVariable column, OrderKind order) {
-        this.column = column;
-        this.order = order;
-    }
-
-    public LogicalVariable getColumn() {
-        return column;
-    }
-
-    public OrderKind getOrder() {
-        return order;
-    }
-
-    public void setColumn(LogicalVariable column) {
-        this.column = column;
-    }
-
-    public void setOrder(OrderKind order) {
-        this.order = order;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof OrderColumn)) {
-            return false;
-        } else {
-            OrderColumn oc = (OrderColumn) obj;
-            return column.equals(oc.getColumn()) && order == oc.getOrder();
-        }
-    }
-
-    @Override
-    public String toString() {
-        return column.toString() + "(" + order + ")";
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
deleted file mode 100644
index 6f88933..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public class OrderedPartitionedProperty implements IPartitioningProperty {
-
-    private ArrayList<OrderColumn> orderColumns;
-    private INodeDomain domain;
-
-    public OrderedPartitionedProperty(ArrayList<OrderColumn> orderColumns, INodeDomain domain) {
-        this.domain = domain;
-        this.orderColumns = orderColumns;
-    }
-
-    public ArrayList<OrderColumn> getOrderColumns() {
-        return orderColumns;
-    }
-
-    public ArrayList<LogicalVariable> getColumns() {
-        ArrayList<LogicalVariable> cols = new ArrayList<LogicalVariable>(orderColumns.size());
-        for (OrderColumn oc : orderColumns) {
-            cols.add(oc.getColumn());
-        }
-        return cols;
-    }
-
-    @Override
-    public PartitioningType getPartitioningType() {
-        return PartitioningType.ORDERED_PARTITIONED;
-    }
-
-    @Override
-    public String toString() {
-        return getPartitioningType().toString() + orderColumns;
-    }
-
-    @Override
-    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
-        orderColumns = PropertiesUtil.replaceOrderColumnsByEqClasses(orderColumns, equivalenceClasses);
-        orderColumns = PropertiesUtil.applyFDsToOrderColumns(orderColumns, fds);
-    }
-
-    @Override
-    public void getColumns(Collection<LogicalVariable> columns) {
-        for (OrderColumn oc : orderColumns) {
-            columns.add(oc.getColumn());
-        }
-    }
-
-    @Override
-    public INodeDomain getNodeDomain() {
-        return domain;
-    }
-
-    @Override
-    public void setNodeDomain(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
deleted file mode 100644
index ce0907f..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-public class PhysicalRequirements {
-    private final IPhysicalPropertiesVector[] requiredProperties;
-    private final IPartitioningRequirementsCoordinator partitioningCoordinator;
-
-    public PhysicalRequirements(IPhysicalPropertiesVector[] requiredProperties,
-            IPartitioningRequirementsCoordinator partitioningCoordinator) {
-        this.requiredProperties = requiredProperties;
-        this.partitioningCoordinator = partitioningCoordinator;
-    }
-
-    public IPhysicalPropertiesVector[] getRequiredProperties() {
-        return requiredProperties;
-    }
-
-    public IPartitioningRequirementsCoordinator getPartitioningCoordinator() {
-        return partitioningCoordinator;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
deleted file mode 100644
index b651e59..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-
-public class PropertiesUtil {
-
-    public Set<LogicalVariable> closureUnderFDs(Collection<LogicalVariable> vars, List<FunctionalDependency> fdList) {
-        Set<LogicalVariable> k = new ListSet<LogicalVariable>(vars);
-        boolean change;
-        do {
-            change = false;
-            for (FunctionalDependency fd : fdList) {
-                List<LogicalVariable> h = fd.getHead();
-                if (k.containsAll(h)) {
-                    List<LogicalVariable> t = fd.getTail();
-                    for (LogicalVariable v : t) {
-                        if (!(k.contains(v))) {
-                            k.add(v);
-                            change = true;
-                        }
-                    }
-                }
-            }
-        } while (change);
-        return k;
-    }
-
-    public static boolean matchLocalProperties(List<ILocalStructuralProperty> reqd,
-            List<ILocalStructuralProperty> dlvd, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
-            List<FunctionalDependency> fds) {
-        if (reqd == null) {
-            return true;
-        }
-        if (dlvd == null) {
-            return false;
-        }
-        normalizeLocals(reqd, equivalenceClasses, fds);
-        normalizeLocals(dlvd, equivalenceClasses, fds);
-
-        ListIterator<ILocalStructuralProperty> dlvdIter = dlvd.listIterator();
-
-        Set<LogicalVariable> rqdCols = new ListSet<LogicalVariable>();
-        Set<LogicalVariable> dlvdCols = new ListSet<LogicalVariable>();
-        for (ILocalStructuralProperty r : reqd) {
-            if (r.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
-                rqdCols.clear();
-                r.getVariables(rqdCols);
-            }
-            boolean implied = false;
-            while (!implied && dlvdIter.hasNext()) {
-                ILocalStructuralProperty d = dlvdIter.next();
-                switch (r.getPropertyType()) {
-                    case LOCAL_ORDER_PROPERTY: {
-                        if (d.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
-                            return false;
-                        }
-                        LocalOrderProperty lop = (LocalOrderProperty) d;
-                        if (lop.implies(r)) {
-                            implied = true;
-                        } else {
-                            return false;
-                        }
-                        break;
-                    }
-                    case LOCAL_GROUPING_PROPERTY: {
-                        dlvdCols.clear();
-                        d.getColumns(dlvdCols);
-                        if (d.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
-                            implied = isPrefixOf(rqdCols.iterator(), dlvdCols.iterator());
-                        } else {
-                            implied = rqdCols.equals(dlvdCols) || isPrefixOf(rqdCols.iterator(), dlvdCols.iterator());
-                        }
-                        break;
-                    }
-                    default: {
-                        throw new IllegalStateException();
-                    }
-                }
-            }
-            if (!implied) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    public static boolean matchPartitioningProps(IPartitioningProperty reqd, IPartitioningProperty dlvd,
-            boolean mayExpandProperties) {
-        INodeDomain dom1 = reqd.getNodeDomain();
-        INodeDomain dom2 = dlvd.getNodeDomain();
-        if (dom1 != null && dom2 != null && !dom1.sameAs(dom2)) {
-            return false;
-        }
-
-        switch (reqd.getPartitioningType()) {
-            case RANDOM: {
-                // anything matches RANDOM
-                return true;
-            }
-            case UNORDERED_PARTITIONED: {
-                switch (dlvd.getPartitioningType()) {
-                    case UNORDERED_PARTITIONED: {
-                        UnorderedPartitionedProperty ur = (UnorderedPartitionedProperty) reqd;
-                        UnorderedPartitionedProperty ud = (UnorderedPartitionedProperty) dlvd;
-                        if (mayExpandProperties) {
-                            return isPrefixOf(ud.getColumnSet().iterator(), ur.getColumnSet().iterator());
-                        } else {
-                            return ur.getColumnSet().equals(ud.getColumnSet());
-                        }
-                    }
-                    case ORDERED_PARTITIONED: {
-                        UnorderedPartitionedProperty ur = (UnorderedPartitionedProperty) reqd;
-                        OrderedPartitionedProperty od = (OrderedPartitionedProperty) dlvd;
-                        if (mayExpandProperties) {
-                            List<LogicalVariable> dlvdSortColumns = orderColumnsToVariables(od.getOrderColumns());
-                            return isPrefixOf(dlvdSortColumns.iterator(), ur.getColumnSet().iterator());
-                        } else {
-                            return ur.getColumnSet().containsAll(od.getOrderColumns())
-                                    && od.getOrderColumns().containsAll(ur.getColumnSet());
-                        }
-                    }
-                    default: {
-                        return false;
-                    }
-                }
-            }
-            case ORDERED_PARTITIONED: {
-                switch (dlvd.getPartitioningType()) {
-                    case ORDERED_PARTITIONED: {
-                        OrderedPartitionedProperty or = (OrderedPartitionedProperty) reqd;
-                        OrderedPartitionedProperty od = (OrderedPartitionedProperty) dlvd;
-                        if (mayExpandProperties) {
-                            return isPrefixOf(od.getOrderColumns().iterator(), or.getOrderColumns().iterator());
-                        } else {
-                            return od.getOrderColumns().equals(or.getOrderColumns());
-                        }
-                    }
-                    default: {
-                        return false;
-                    }
-                }
-            }
-            default: {
-                return (dlvd.getPartitioningType() == reqd.getPartitioningType());
-            }
-        }
-    }
-
-    /**
-     * Converts a list of OrderColumns to a list of LogicalVariables.
-     * 
-     * @param orderColumns
-     *            , a list of OrderColumns
-     * @return the list of LogicalVariables
-     */
-    private static List<LogicalVariable> orderColumnsToVariables(List<OrderColumn> orderColumns) {
-        List<LogicalVariable> columns = new ArrayList<LogicalVariable>();
-        for (OrderColumn oc : orderColumns) {
-            columns.add(oc.getColumn());
-        }
-        return columns;
-    }
-
-    /**
-     * @param pref
-     * @param target
-     * @return true iff pref is a prefix of target
-     */
-    private static <T> boolean isPrefixOf(Iterator<T> pref, Iterator<T> target) {
-        while (pref.hasNext()) {
-            T v = pref.next();
-            if (!target.hasNext()) {
-                return false;
-            }
-            if (!v.equals(target.next())) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    public static ArrayList<OrderColumn> applyFDsToOrderColumns(ArrayList<OrderColumn> orderColumns,
-            List<FunctionalDependency> fds) {
-        // the set of vars. is ordered
-        // so we try the variables in order from last to first
-        if (fds == null || fds.isEmpty()) {
-            return orderColumns;
-        }
-
-        int deleted = 0;
-        for (int i = orderColumns.size() - 1; i >= 0; i--) {
-            for (FunctionalDependency fdep : fds) {
-                if (impliedByPrefix(orderColumns, i, fdep)) {
-                    orderColumns.set(i, null);
-                    deleted++;
-                    break;
-                }
-            }
-        }
-        ArrayList<OrderColumn> norm = new ArrayList<OrderColumn>(orderColumns.size() - deleted);
-        for (OrderColumn oc : orderColumns) {
-            if (oc != null) {
-                norm.add(oc);
-            }
-        }
-        return norm;
-    }
-
-    public static ArrayList<OrderColumn> replaceOrderColumnsByEqClasses(ArrayList<OrderColumn> orderColumns,
-            Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
-        if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
-            return orderColumns;
-        }
-        ArrayList<OrderColumn> norm = new ArrayList<OrderColumn>();
-        for (OrderColumn v : orderColumns) {
-            EquivalenceClass ec = equivalenceClasses.get(v.getColumn());
-            if (ec == null) {
-                norm.add(v);
-            } else {
-                if (ec.representativeIsConst()) {
-                    // trivially satisfied, so the var. can be removed
-                } else {
-                    norm.add(new OrderColumn(ec.getVariableRepresentative(), v.getOrder()));
-                }
-            }
-        }
-        return norm;
-    }
-
-    private static boolean impliedByPrefix(ArrayList<OrderColumn> vars, int i, FunctionalDependency fdep) {
-        if (!fdep.getTail().contains(vars.get(i).getColumn())) {
-            return false;
-        }
-        boolean fdSat = true;
-        for (LogicalVariable pv : fdep.getHead()) {
-            boolean isInPrefix = false;
-            for (int j = 0; j < i; j++) {
-                if (vars.get(j).getColumn().equals(pv)) {
-                    isInPrefix = true;
-                    break;
-                }
-            }
-            if (!isInPrefix) {
-                fdSat = false;
-                break;
-            }
-        }
-        return fdSat;
-    }
-
-    private static void normalizeLocals(List<ILocalStructuralProperty> props,
-            Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
-        ListIterator<ILocalStructuralProperty> propIter = props.listIterator();
-        int pos = -1;
-        while (propIter.hasNext()) {
-            ILocalStructuralProperty p = propIter.next();
-            if (p.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
-                ((LocalGroupingProperty) p).normalizeGroupingColumns(equivalenceClasses, fds);
-                pos++;
-            } else {
-                ((LocalOrderProperty) p).normalizeOrderingColumns(equivalenceClasses, fds);
-                pos++;
-            }
-        }
-
-        if (pos < 1) {
-            return;
-        }
-
-        while (propIter.hasPrevious()) {
-            ILocalStructuralProperty p = propIter.previous();
-            ListIterator<ILocalStructuralProperty> secondIter = props.listIterator(pos);
-            pos--;
-            Set<LogicalVariable> cols = new ListSet<LogicalVariable>();
-            while (secondIter.hasPrevious()) {
-                secondIter.previous().getColumns(cols);
-            }
-            secondIter = null;
-            for (FunctionalDependency fdep : fds) {
-                LinkedList<LogicalVariable> columnsOfP = new LinkedList<LogicalVariable>();
-                p.getColumns(columnsOfP);
-                if (impliedByPrefix(columnsOfP, cols, fdep)) {
-                    propIter.remove();
-                    break;
-                }
-            }
-        }
-    }
-
-    private static boolean impliedByPrefix(List<LogicalVariable> colsOfProp, Set<LogicalVariable> colsOfPrefix,
-            FunctionalDependency fdep) {
-        return fdep.getTail().containsAll(colsOfProp) && colsOfPrefix.containsAll(fdep.getHead());
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
deleted file mode 100644
index 06008a6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public class RandomPartitioningProperty implements IPartitioningProperty {
-
-    private INodeDomain domain;
-
-    public RandomPartitioningProperty(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-    @Override
-    public PartitioningType getPartitioningType() {
-        return PartitioningType.RANDOM;
-    }
-
-    @Override
-    public String toString() {
-        return getPartitioningType() + " domain:" + domain;
-    }
-
-    @Override
-    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
-        // do nothing
-    }
-
-    @Override
-    public void getColumns(Collection<LogicalVariable> columns) {
-    }
-
-    @Override
-    public INodeDomain getNodeDomain() {
-        return domain;
-    }
-
-    @Override
-    public void setNodeDomain(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
deleted file mode 100644
index 688523b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ResultSetDomain.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-public class ResultSetDomain implements INodeDomain {
-    @Override
-    public boolean sameAs(INodeDomain domain) {
-        return true;
-    }
-
-    @Override
-    public Integer cardinality() {
-        return 0;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
deleted file mode 100644
index 7290319..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public class StructuralPropertiesVector implements IPhysicalPropertiesVector {
-    private List<ILocalStructuralProperty> propsLocal;
-    private IPartitioningProperty propPartitioning;
-
-    public static final StructuralPropertiesVector EMPTY_PROPERTIES_VECTOR = new StructuralPropertiesVector(null,
-            new ArrayList<ILocalStructuralProperty>());
-
-    public StructuralPropertiesVector(IPartitioningProperty propPartitioning, List<ILocalStructuralProperty> propsLocal) {
-        this.propPartitioning = propPartitioning;
-        this.propsLocal = propsLocal;
-    }
-
-    @Override
-    public String toString() {
-        return "propsLocal=" + propsLocal + "\tpropPartioning=" + propPartitioning;
-    }
-
-    @Override
-    public IPartitioningProperty getPartitioningProperty() {
-        return propPartitioning;
-    }
-
-    @Override
-    public List<ILocalStructuralProperty> getLocalProperties() {
-        return propsLocal;
-    }
-
-    @Override
-    public IPhysicalPropertiesVector clone() {
-        List<ILocalStructuralProperty> propsCopy = new LinkedList<ILocalStructuralProperty>();
-        if (propsLocal != null) {
-            propsCopy.addAll(propsLocal);
-        }
-        return new StructuralPropertiesVector(propPartitioning, propsCopy);
-    }
-
-    /**
-     * 
-     * @param reqd
-     *            vector of required properties
-     * @return a vector of properties from pvector that are not delivered by the
-     *         current vector or null if none
-     */
-    @Override
-    public IPhysicalPropertiesVector getUnsatisfiedPropertiesFrom(IPhysicalPropertiesVector reqd,
-            boolean mayExpandProperties, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
-            List<FunctionalDependency> fds) {
-        List<ILocalStructuralProperty> plist = reqd.getLocalProperties();
-        List<ILocalStructuralProperty> diffLocals = null;
-        if (plist != null && !plist.isEmpty()) {
-            if (!PropertiesUtil.matchLocalProperties(plist, propsLocal, equivalenceClasses, fds)) {
-                diffLocals = plist;
-            }
-        }
-
-        IPartitioningProperty diffPart = null;
-        IPartitioningProperty reqdPart = reqd.getPartitioningProperty();
-        if (reqdPart != null) {
-            if (mayExpandProperties) {
-                reqdPart.normalize(equivalenceClasses, fds);
-            } else {
-                reqdPart.normalize(equivalenceClasses, null);
-            }
-            propPartitioning.normalize(equivalenceClasses, fds);
-            if (!PropertiesUtil.matchPartitioningProps(reqdPart, propPartitioning, mayExpandProperties)) {
-                diffPart = reqdPart;
-            }
-        }
-
-        if (diffLocals == null && diffPart == null) {
-            return null;
-        } else {
-            return new StructuralPropertiesVector(diffPart, diffLocals);
-        }
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
deleted file mode 100644
index e27edc5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
-
-public abstract class TypePropagationPolicy {
-    public static final TypePropagationPolicy ALL = new TypePropagationPolicy() {
-
-        @Override
-        public Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
-                List<LogicalVariable> nonNullVariableList, List<List<LogicalVariable>> correlatedNullableVariableLists,
-                ITypeEnvPointer... typeEnvs) throws AlgebricksException {
-            for (ITypeEnvPointer p : typeEnvs) {
-                IVariableTypeEnvironment env = p.getTypeEnv();
-                if (env == null) {
-                    throw new AlgebricksException("Null environment for pointer " + p + " in getVarType for var=" + var);
-                }
-                Object t = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists);
-                if (t != null) {
-                    if (ntc != null && ntc.canBeNull(t)) {
-                        for (List<LogicalVariable> list : correlatedNullableVariableLists) {
-                            if (list.contains(var)) {
-                                for (LogicalVariable v : list) {
-                                    if (nonNullVariableList.contains(v)) {
-                                        return ntc.getNonOptionalType(t);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    return t;
-                }
-            }
-            return null;
-        }
-    };
-
-    public static final TypePropagationPolicy LEFT_OUTER = new TypePropagationPolicy() {
-
-        @Override
-        public Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
-                List<LogicalVariable> nonNullVariableList, List<List<LogicalVariable>> correlatedNullableVariableLists,
-                ITypeEnvPointer... typeEnvs) throws AlgebricksException {
-            int n = typeEnvs.length;
-            for (int i = 0; i < n; i++) {
-                Object t = typeEnvs[i].getTypeEnv().getVarType(var, nonNullVariableList,
-                        correlatedNullableVariableLists);
-                if (t != null) {
-                    if (i == 0) { // inner branch
-                        return t;
-                    } else { // outer branch
-                        boolean nonNullVarIsProduced = false;
-                        for (LogicalVariable v : nonNullVariableList) {
-                            if (v == var) {
-                                nonNullVarIsProduced = true;
-                                break;
-                            }
-                            if (typeEnvs[i].getTypeEnv().getVarType(v) != null) {
-                                nonNullVarIsProduced = true;
-                                break;
-                            }
-                        }
-                        if (nonNullVarIsProduced) {
-                            return t;
-                        } else {
-                            return ntc.makeNullableType(t);
-                        }
-                    }
-                }
-            }
-            return null;
-        }
-    };
-
-    public abstract Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
-            List<LogicalVariable> nonNullVariableList, List<List<LogicalVariable>> correlatedNullableVariableLists,
-            ITypeEnvPointer... typeEnvs) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
deleted file mode 100644
index 3e79192..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public final class UnorderedPartitionedProperty extends AbstractGroupingProperty implements IPartitioningProperty {
-
-    private INodeDomain domain;
-
-    public UnorderedPartitionedProperty(Set<LogicalVariable> partitioningVariables, INodeDomain domain) {
-        super(partitioningVariables);
-        this.domain = domain;
-    }
-
-    @Override
-    public PartitioningType getPartitioningType() {
-        return PartitioningType.UNORDERED_PARTITIONED;
-    }
-
-    @Override
-    public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
-        normalizeGroupingColumns(equivalenceClasses, fds);
-    }
-
-    @Override
-    public String toString() {
-        return getPartitioningType().toString() + columnSet;
-    }
-
-    @Override
-    public void getColumns(Collection<LogicalVariable> columns) {
-        columns.addAll(columnSet);
-    }
-
-    @Override
-    public INodeDomain getNodeDomain() {
-        return domain;
-    }
-
-    @Override
-    public void setNodeDomain(INodeDomain domain) {
-        this.domain = domain;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
deleted file mode 100644
index 7a47c1e..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-
-public class UnpartitionedPropertyComputer implements IPropertiesComputer {
-
-    public static final UnpartitionedPropertyComputer INSTANCE = new UnpartitionedPropertyComputer();
-
-    private UnpartitionedPropertyComputer() {
-    }
-
-    @Override
-    public IPartitioningProperty computePartitioningProperty(ILogicalExpression expr) {
-        return IPartitioningProperty.UNPARTITIONED;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
deleted file mode 100644
index 5ac3a93..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-
-public abstract class VariablePropagationPolicy {
-    public static final VariablePropagationPolicy ALL = new VariablePropagationPolicy() {
-        @Override
-        public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
-            int n = sources.length;
-            for (int i = 0; i < n; i++) {
-                target.addAllNewVariables(sources[i]);
-            }
-        }
-    };
-
-    public static final VariablePropagationPolicy NONE = new VariablePropagationPolicy() {
-        @Override
-        public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
-            // do nothing
-        }
-    };
-
-    /**
-     * Adds, from each source, only variables that do not already appear in the
-     * target.
-     * 
-     * 
-     */
-    public static final VariablePropagationPolicy ADDNEWVARIABLES = new VariablePropagationPolicy() {
-        @Override
-        public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
-            for (IOperatorSchema s : sources) {
-                for (LogicalVariable v : s) {
-                    if (target.findVariable(v) < 0) {
-                        target.addVariable(v);
-                    }
-                }
-            }
-        }
-    };
-
-    public abstract void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-            throws AlgebricksException;
-
-    public static VariablePropagationPolicy concat(final VariablePropagationPolicy... policies) {
-        return new VariablePropagationPolicy() {
-            @Override
-            public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
-                    throws AlgebricksException {
-                if (policies.length != sources.length) {
-                    throw new IllegalArgumentException();
-                }
-                for (int i = 0; i < policies.length; ++i) {
-                    VariablePropagationPolicy p = policies[i];
-                    IOperatorSchema s = sources[i];
-                    p.propagateVariables(target, s);
-                }
-            }
-        };
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
deleted file mode 100644
index f3724a5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.scripting;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IScriptDescription {
-    public enum ScriptKind {
-        STRING_STREAMING
-    }
-
-    public ScriptKind getKind();
-
-    public List<Pair<LogicalVariable, Object>> getVarTypePairs();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
deleted file mode 100644
index f030faf..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.scripting;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
-
-public class StringStreamingScriptDescription implements IScriptDescription {
-
-    private final String command;
-    private final IPrinterFactory[] printerFactories;
-    private final char fieldDelimiter;
-    private final ITupleParserFactory parserFactory;
-    private final List<Pair<LogicalVariable, Object>> varTypePairs;
-
-    public StringStreamingScriptDescription(String command, IPrinterFactory[] printerFactories, char fieldDelimiter,
-            ITupleParserFactory parserFactory, List<Pair<LogicalVariable, Object>> varTypePairs) {
-        this.command = command;
-        this.printerFactories = printerFactories;
-        this.fieldDelimiter = fieldDelimiter;
-        this.parserFactory = parserFactory;
-        this.varTypePairs = varTypePairs;
-    }
-
-    @Override
-    public ScriptKind getKind() {
-        return ScriptKind.STRING_STREAMING;
-    }
-
-    public String getCommand() {
-        return command;
-    }
-
-    public IPrinterFactory[] getPrinterFactories() {
-        return printerFactories;
-    }
-
-    public char getFieldDelimiter() {
-        return fieldDelimiter;
-    }
-
-    public ITupleParserFactory getParserFactory() {
-        return parserFactory;
-    }
-
-    @Override
-    public List<Pair<LogicalVariable, Object>> getVarTypePairs() {
-        return varTypePairs;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
deleted file mode 100644
index 9833c17..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public abstract class AbstractTypeEnvironment implements IVariableTypeEnvironment {
-
-    protected final Map<LogicalVariable, Object> varTypeMap = new HashMap<LogicalVariable, Object>();
-    protected final IExpressionTypeComputer expressionTypeComputer;
-    protected final IMetadataProvider<?, ?> metadataProvider;
-
-    public AbstractTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
-            IMetadataProvider<?, ?> metadataProvider) {
-        this.expressionTypeComputer = expressionTypeComputer;
-        this.metadataProvider = metadataProvider;
-    }
-
-    @Override
-    public Object getType(ILogicalExpression expr) throws AlgebricksException {
-        return expressionTypeComputer.getType(expr, metadataProvider, this);
-    }
-
-    @Override
-    public void setVarType(LogicalVariable var, Object type) {
-        varTypeMap.put(var, type);
-    }
-
-    @Override
-    public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException {
-        Object t = varTypeMap.get(v1);
-        if (t == null) {
-            return false;
-        }
-        varTypeMap.put(v1, null);
-        varTypeMap.put(v2, t);
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
deleted file mode 100644
index 0b56db7..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-
-public interface ITypeEnvPointer {
-    public IVariableTypeEnvironment getTypeEnv();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java
deleted file mode 100644
index 7945fa7..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public interface ITypingContext {
-    public abstract IVariableTypeEnvironment getOutputTypeEnvironment(ILogicalOperator op);
-
-    public abstract void setOutputTypeEnvironment(ILogicalOperator op, IVariableTypeEnvironment env);
-
-    public abstract IExpressionTypeComputer getExpressionTypeComputer();
-
-    public abstract INullableTypeComputer getNullableTypeComputer();
-
-    public abstract IMetadataProvider<?, ?> getMetadataProvider();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
deleted file mode 100644
index eb0eb71..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public class NonPropagatingTypeEnvironment extends AbstractTypeEnvironment {
-
-    public NonPropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
-            IMetadataProvider<?, ?> metadataProvider) {
-        super(expressionTypeComputer, metadataProvider);
-    }
-
-    @Override
-    public Object getVarType(LogicalVariable var) throws AlgebricksException {
-        return varTypeMap.get(var);
-    }
-
-    @Override
-    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
-            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
-        return getVarType(var);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
deleted file mode 100644
index 6a1c8b5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-
-public class OpRefTypeEnvPointer implements ITypeEnvPointer {
-
-    private final Mutable<ILogicalOperator> op;
-    private final ITypingContext ctx;
-
-    public OpRefTypeEnvPointer(Mutable<ILogicalOperator> op, ITypingContext ctx) {
-        this.op = op;
-        this.ctx = ctx;
-    }
-
-    @Override
-    public IVariableTypeEnvironment getTypeEnv() {
-        return ctx.getOutputTypeEnvironment(op.getValue());
-    }
-
-    @Override
-    public String toString() {
-        return this.getClass().getName() + ":" + op;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
deleted file mode 100644
index 87ee385..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public class PropagateOperatorInputsTypeEnvironment extends AbstractTypeEnvironment {
-
-    private final List<LogicalVariable> nonNullVariables = new ArrayList<LogicalVariable>();
-    private final List<List<LogicalVariable>> correlatedNullableVariableLists = new ArrayList<List<LogicalVariable>>();
-    private final ILogicalOperator op;
-    private final ITypingContext ctx;
-
-    public PropagateOperatorInputsTypeEnvironment(ILogicalOperator op, ITypingContext ctx,
-            IExpressionTypeComputer expressionTypeComputer, IMetadataProvider<?, ?> metadataProvider) {
-        super(expressionTypeComputer, metadataProvider);
-        this.op = op;
-        this.ctx = ctx;
-    }
-
-    public List<LogicalVariable> getNonNullVariables() {
-        return nonNullVariables;
-    }
-
-    @Override
-    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
-            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
-        nonNullVariableList.addAll(nonNullVariables);
-        return getVarTypeFullList(var, nonNullVariableList, correlatedNullableVariableLists);
-    }
-
-    private Object getVarTypeFullList(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
-            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
-        Object t = varTypeMap.get(var);
-        if (t != null) {
-            return t;
-        }
-        for (Mutable<ILogicalOperator> r : op.getInputs()) {
-            ILogicalOperator c = r.getValue();
-            IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(c);
-            Object t2 = env.getVarType(var, nonNullVariableList, correlatedNullableVariableLists);
-            if (t2 != null) {
-                return t2;
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Object getVarType(LogicalVariable var) throws AlgebricksException {
-        return getVarTypeFullList(var, nonNullVariables, correlatedNullableVariableLists);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
deleted file mode 100644
index d167cd7..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
-
-public class PropagatingTypeEnvironment extends AbstractTypeEnvironment {
-
-    private final TypePropagationPolicy policy;
-
-    private final INullableTypeComputer nullableTypeComputer;
-
-    private final ITypeEnvPointer[] envPointers;
-
-    private final List<LogicalVariable> nonNullVariables = new ArrayList<LogicalVariable>();
-
-    private final List<List<LogicalVariable>> correlatedNullableVariableLists = new ArrayList<List<LogicalVariable>>();
-
-    public PropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
-            INullableTypeComputer nullableTypeComputer, IMetadataProvider<?, ?> metadataProvider,
-            TypePropagationPolicy policy, ITypeEnvPointer[] envPointers) {
-        super(expressionTypeComputer, metadataProvider);
-        this.nullableTypeComputer = nullableTypeComputer;
-        this.policy = policy;
-        this.envPointers = envPointers;
-    }
-
-    @Override
-    public Object getVarType(LogicalVariable var) throws AlgebricksException {
-        return getVarTypeFullList(var, nonNullVariables, correlatedNullableVariableLists);
-    }
-
-    public List<LogicalVariable> getNonNullVariables() {
-        return nonNullVariables;
-    }
-
-    public List<List<LogicalVariable>> getCorrelatedNullableVariableLists() {
-        return correlatedNullableVariableLists;
-    }
-
-    @Override
-    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
-            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
-        for (LogicalVariable v : nonNullVariables) {
-            if (!nonNullVariableList.contains(v)) {
-                nonNullVariableList.add(v);
-            }
-        }
-        Object t = getVarTypeFullList(var, nonNullVariableList, correlatedNullableVariableLists);
-        for (List<LogicalVariable> list : this.correlatedNullableVariableLists) {
-            if (!correlatedNullableVariableLists.contains(list)) {
-                correlatedNullableVariableLists.add(list);
-            }
-        }
-        return t;
-    }
-
-    private Object getVarTypeFullList(LogicalVariable var, List<LogicalVariable> nonNullVariableList,
-            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException {
-        Object t = varTypeMap.get(var);
-        if (t != null) {
-            return t;
-        }
-        return policy.getVarType(var, nullableTypeComputer, nonNullVariableList, correlatedNullableVariableLists,
-                envPointers);
-    }
-}

[06/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
new file mode 100644
index 0000000..b268e77
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PullSelectOutOfEqJoin implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+        if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
+
+        ILogicalExpression expr = join.getCondition().getValue();
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
+        FunctionIdentifier fi = fexp.getFunctionIdentifier();
+        if (!fi.equals(AlgebricksBuiltinFunctions.AND)) {
+            return false;
+        }
+        List<Mutable<ILogicalExpression>> eqVarVarComps = new ArrayList<Mutable<ILogicalExpression>>();
+        List<Mutable<ILogicalExpression>> otherPredicates = new ArrayList<Mutable<ILogicalExpression>>();
+        for (Mutable<ILogicalExpression> arg : fexp.getArguments()) {
+            if (isEqVarVar(arg.getValue())) {
+                eqVarVarComps.add(arg);
+            } else {
+                otherPredicates.add(arg);
+            }
+        }
+        if (eqVarVarComps.isEmpty() || otherPredicates.isEmpty()) {
+            return false;
+        }
+        // pull up
+        ILogicalExpression pulledCond = makeCondition(otherPredicates, context);
+        SelectOperator select = new SelectOperator(new MutableObject<ILogicalExpression>(pulledCond), false, null);
+        ILogicalExpression newJoinCond = makeCondition(eqVarVarComps, context);
+        join.getCondition().setValue(newJoinCond);
+        select.getInputs().add(new MutableObject<ILogicalOperator>(join));
+        opRef.setValue(select);
+        context.computeAndSetTypeEnvironmentForOperator(select);
+        return true;
+    }
+
+    private ILogicalExpression makeCondition(List<Mutable<ILogicalExpression>> predList, IOptimizationContext context) {
+        if (predList.size() > 1) {
+            IFunctionInfo finfo = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND);
+            return new ScalarFunctionCallExpression(finfo, predList);
+        } else {
+            return predList.get(0).getValue();
+        }
+    }
+
+    private boolean isEqVarVar(ILogicalExpression expr) {
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+        if (!f.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.EQ)) {
+            return false;
+        }
+        ILogicalExpression e1 = f.getArguments().get(0).getValue();
+        if (e1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            return false;
+        } else {
+            ILogicalExpression e2 = f.getArguments().get(1).getValue();
+            return e2.getExpressionTag() == LogicalExpressionTag.VARIABLE;
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java
new file mode 100644
index 0000000..418ee36
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Pushes an AssignOperator below a UnionAll operator by creating an new AssignOperator below each of
+ * the UnionAllOperator's branches with appropriate variable replacements.
+ * This rule can help to enable other rules that are difficult to fire across a UnionAllOperator,
+ * for example, eliminating common sub-expressions.
+ * Example:
+ * Before plan:
+ * ...
+ * assign [$$20, $$21] <- [funcA($$3), funcB($$6)]
+ * union ($$1, $$2, $$3) ($$4, $$5, $$6)
+ * union_branch_0
+ * ...
+ * union_branch_1
+ * ...
+ * After plan:
+ * ...
+ * union ($$1, $$2, $$3) ($$4, $$5, $$6) ($$22, $$24, $$20) ($$23, $$25, $$21)
+ * assign [$$22, $$23] <- [funcA($$1), funcB($$4)]
+ * union_branch_0
+ * ...
+ * assign [$$24, $$25] <- [funcA($$2), funcB($$5)]
+ * union_branch_1
+ * ...
+ */
+public class PushAssignBelowUnionAllRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (!op.hasInputs()) {
+            return false;
+        }
+
+        boolean modified = false;
+        for (int i = 0; i < op.getInputs().size(); i++) {
+            AbstractLogicalOperator childOp = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
+            if (childOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+                continue;
+            }
+            AssignOperator assignOp = (AssignOperator) childOp;
+            for (Mutable<ILogicalExpression> expr : assignOp.getExpressions()) {
+                if (!expr.getValue().isFunctional()) {
+                    return false;
+                }
+            }
+
+            AbstractLogicalOperator childOfChildOp = (AbstractLogicalOperator) assignOp.getInputs().get(0).getValue();
+            if (childOfChildOp.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
+                continue;
+            }
+            UnionAllOperator unionOp = (UnionAllOperator) childOfChildOp;
+
+            Set<LogicalVariable> assignUsedVars = new HashSet<LogicalVariable>();
+            VariableUtilities.getUsedVariables(assignOp, assignUsedVars);
+
+            List<LogicalVariable> assignVars = assignOp.getVariables();
+
+            AssignOperator[] newAssignOps = new AssignOperator[2];
+            for (int j = 0; j < unionOp.getInputs().size(); j++) {
+                newAssignOps[j] = createAssignBelowUnionAllBranch(unionOp, j, assignOp, assignUsedVars, context);
+            }
+            // Add original assign variables to the union variable mappings.
+            for (int j = 0; j < assignVars.size(); j++) {
+                LogicalVariable first = newAssignOps[0].getVariables().get(j);
+                LogicalVariable second = newAssignOps[1].getVariables().get(j);
+                Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
+                        first, second, assignVars.get(j));
+                unionOp.getVariableMappings().add(varMapping);
+            }
+            context.computeAndSetTypeEnvironmentForOperator(unionOp);
+
+            // Remove original assign operator.
+            op.getInputs().set(i, assignOp.getInputs().get(0));
+            context.computeAndSetTypeEnvironmentForOperator(op);
+            modified = true;
+        }
+
+        return modified;
+    }
+
+    private AssignOperator createAssignBelowUnionAllBranch(UnionAllOperator unionOp, int inputIndex,
+            AssignOperator originalAssignOp, Set<LogicalVariable> assignUsedVars, IOptimizationContext context)
+            throws AlgebricksException {
+        AssignOperator newAssignOp = cloneAssignOperator(originalAssignOp, context);
+        newAssignOp.getInputs()
+                .add(new MutableObject<ILogicalOperator>(unionOp.getInputs().get(inputIndex).getValue()));
+        context.computeAndSetTypeEnvironmentForOperator(newAssignOp);
+        unionOp.getInputs().get(inputIndex).setValue(newAssignOp);
+        int numVarMappings = unionOp.getVariableMappings().size();
+        for (int i = 0; i < numVarMappings; i++) {
+            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping = unionOp.getVariableMappings().get(i);
+            if (assignUsedVars.contains(varMapping.third)) {
+                LogicalVariable replacementVar;
+                if (inputIndex == 0) {
+                    replacementVar = varMapping.first;
+                } else {
+                    replacementVar = varMapping.second;
+                }
+                VariableUtilities.substituteVariables(newAssignOp, varMapping.third, replacementVar, context);
+            }
+        }
+        return newAssignOp;
+    }
+
+    /**
+     * Clones the given assign operator changing the returned variables to be new ones.
+     * Also, leaves the inputs of the clone clear.
+     */
+    private AssignOperator cloneAssignOperator(AssignOperator assignOp, IOptimizationContext context) {
+        List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+        List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+        int numVars = assignOp.getVariables().size();
+        for (int i = 0; i < numVars; i++) {
+            vars.add(context.newVar());
+            exprs.add(new MutableObject<ILogicalExpression>(assignOp.getExpressions().get(i).getValue()
+                    .cloneExpression()));
+        }
+        AssignOperator assignCloneOp = new AssignOperator(vars, exprs);
+        assignCloneOp.setExecutionMode(assignOp.getExecutionMode());
+        return assignCloneOp;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
new file mode 100644
index 0000000..d8523f6
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushAssignDownThroughProductRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+        if (join.getCondition().getValue() != ConstantExpression.TRUE) {
+            return false;
+        }
+
+        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op1, used);
+
+        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+        ILogicalOperator b0 = b0Ref.getValue();
+        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(b0, b0Scm);
+        if (b0Scm.containsAll(used)) {
+            // push assign on left branch
+            op2Ref.setValue(b0);
+            b0Ref.setValue(op1);
+            opRef.setValue(op2);
+            return true;
+        } else {
+            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+            ILogicalOperator b1 = b1Ref.getValue();
+            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(b1, b1Scm);
+            if (b1Scm.containsAll(used)) {
+                // push assign on right branch
+                op2Ref.setValue(b1);
+                b1Ref.setValue(op1);
+                opRef.setValue(op2);
+                return true;
+            } else {
+                return false;
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java
new file mode 100644
index 0000000..0d24618
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java
@@ -0,0 +1,208 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Pushes function-call expressions below a join if possible.
+ * Assigns the result of such function-calls expressions to new variables, and replaces the original
+ * expression with a corresponding variable reference expression.
+ * This rule can help reduce the cost of computing expensive functions by pushing them below
+ * a join (which may blow up the cardinality).
+ * Also, this rule may help to enable other rules such as common subexpression elimination, again to reduce
+ * the number of calls to expensive functions.
+ * 
+ * Example: (we are pushing pushMeFunc)
+ * 
+ * Before plan:
+ * assign [$$10] <- [funcA(funcB(pushMeFunc($$3, $$4)))]
+ *   join (some condition) 
+ *     join_branch_0 where $$3 and $$4 are not live
+ *       ...
+ *     join_branch_1 where $$3 and $$4 are live
+ *       ...
+ * 
+ * After plan:
+ * assign [$$10] <- [funcA(funcB($$11))]
+ *   join (some condition) 
+ *     join_branch_0 where $$3 and $$4 are not live
+ *       ...
+ *     join_branch_1 where $$3 and $$4 are live
+ *       assign[$$11] <- [pushMeFunc($$3, $$4)]
+ *         ...
+ */
+public class PushFunctionsBelowJoin implements IAlgebraicRewriteRule {
+
+    private final Set<FunctionIdentifier> toPushFuncIdents;
+    private final List<Mutable<ILogicalExpression>> funcExprs = new ArrayList<Mutable<ILogicalExpression>>();
+    private final List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+    private final List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+
+    public PushFunctionsBelowJoin(Set<FunctionIdentifier> toPushFuncIdents) {
+        this.toPushFuncIdents = toPushFuncIdents;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator assignOp = (AssignOperator) op;
+
+        // Find a join operator below this assign.
+        Mutable<ILogicalOperator> joinOpRef = findJoinOp(assignOp.getInputs().get(0));
+        if (joinOpRef == null) {
+            return false;
+        }
+        AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinOpRef.getValue();
+
+        // Check if the assign uses a function that we wish to push below the join if possible.
+        funcExprs.clear();
+        gatherFunctionCalls(assignOp, funcExprs);
+        if (funcExprs.isEmpty()) {
+            return false;
+        }
+
+        // Try to push the functions down the input branches of the join.
+        boolean modified = false;
+        if (pushDownFunctions(joinOp, 0, funcExprs, context)) {
+            modified = true;
+        }
+        if (pushDownFunctions(joinOp, 1, funcExprs, context)) {
+            modified = true;
+        }
+        if (modified) {
+            context.computeAndSetTypeEnvironmentForOperator(joinOp);
+        }
+        return modified;
+    }
+
+    private Mutable<ILogicalOperator> findJoinOp(Mutable<ILogicalOperator> opRef) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        switch (op.getOperatorTag()) {
+            case INNERJOIN:
+            case LEFTOUTERJOIN: {
+                return opRef;
+            }
+            // Bail on these operators.
+            case GROUP:
+            case AGGREGATE:
+            case DISTINCT:
+            case UNNEST_MAP: {
+                return null;
+            }
+            // Traverse children.
+            default: {
+                for (Mutable<ILogicalOperator> childOpRef : op.getInputs()) {
+                    return findJoinOp(childOpRef);
+                }
+            }
+        }
+        return null;
+    }
+
+    private void gatherFunctionCalls(AssignOperator assignOp, List<Mutable<ILogicalExpression>> funcExprs) {
+        for (Mutable<ILogicalExpression> exprRef : assignOp.getExpressions()) {
+            gatherFunctionCalls(exprRef, funcExprs);
+        }
+    }
+
+    private void gatherFunctionCalls(Mutable<ILogicalExpression> exprRef, List<Mutable<ILogicalExpression>> funcExprs) {
+        AbstractLogicalExpression expr = (AbstractLogicalExpression) exprRef.getValue();
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return;
+        }
+        // Check whether the function is a function we want to push.
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+        if (toPushFuncIdents.contains(funcExpr.getFunctionIdentifier())) {
+            funcExprs.add(exprRef);
+        }
+        // Traverse arguments.
+        for (Mutable<ILogicalExpression> funcArg : funcExpr.getArguments()) {
+            gatherFunctionCalls(funcArg, funcExprs);
+        }
+    }
+
+    private boolean pushDownFunctions(AbstractBinaryJoinOperator joinOp, int inputIndex,
+            List<Mutable<ILogicalExpression>> funcExprs, IOptimizationContext context) throws AlgebricksException {
+        ILogicalOperator joinInputOp = joinOp.getInputs().get(inputIndex).getValue();
+        liveVars.clear();
+        VariableUtilities.getLiveVariables(joinInputOp, liveVars);
+        Iterator<Mutable<ILogicalExpression>> funcIter = funcExprs.iterator();
+        List<LogicalVariable> assignVars = null;
+        List<Mutable<ILogicalExpression>> assignExprs = null;
+        while (funcIter.hasNext()) {
+            Mutable<ILogicalExpression> funcExprRef = funcIter.next();
+            ILogicalExpression funcExpr = funcExprRef.getValue();
+            usedVars.clear();
+            funcExpr.getUsedVariables(usedVars);
+            // Check if we can push the function down this branch.
+            if (liveVars.containsAll(usedVars)) {
+                if (assignVars == null) {
+                    assignVars = new ArrayList<LogicalVariable>();
+                    assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
+                }
+                // Replace the original expression with a variable reference expression.
+                LogicalVariable replacementVar = context.newVar();
+                assignVars.add(replacementVar);
+                assignExprs.add(new MutableObject<ILogicalExpression>(funcExpr));
+                funcExprRef.setValue(new VariableReferenceExpression(replacementVar));
+                funcIter.remove();
+            }
+        }
+        // Create new assign operator below the join if any functions can be pushed.
+        if (assignVars != null) {
+            AssignOperator newAssign = new AssignOperator(assignVars, assignExprs);
+            newAssign.getInputs().add(new MutableObject<ILogicalOperator>(joinInputOp));
+            newAssign.setExecutionMode(joinOp.getExecutionMode());
+            joinOp.getInputs().get(inputIndex).setValue(newAssign);
+            context.computeAndSetTypeEnvironmentForOperator(newAssign);
+            return true;
+        }
+        return false;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java
new file mode 100644
index 0000000..56b2a8e
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushGroupByIntoSortRule.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * @author yingyib
+ *         merge externalsort+preclustered-gby into sort-gby
+ */
+public class PushGroupByIntoSortRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator op1 = opRef.getValue();
+        if (op1 == null) {
+            return false;
+        }
+        boolean changed = false;
+        for (Mutable<ILogicalOperator> childRef : op1.getInputs()) {
+            AbstractLogicalOperator op = (AbstractLogicalOperator) childRef.getValue();
+            if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
+                PhysicalOperatorTag opTag = op.getPhysicalOperator().getOperatorTag();
+                GroupByOperator groupByOperator = (GroupByOperator) op;
+                if (opTag == PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
+                    Mutable<ILogicalOperator> op2Ref = op.getInputs().get(0).getValue().getInputs().get(0);
+                    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+                    if (op2.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT) {
+                        AbstractStableSortPOperator sortPhysicalOperator = (AbstractStableSortPOperator) op2
+                                .getPhysicalOperator();
+                        if (groupByOperator.getNestedPlans().size() != 1) {
+                            //Sort group-by currently works only for one nested plan with one root containing
+                            //an aggregate and a nested-tuple-source.
+                            continue;
+                        }
+                        ILogicalPlan p0 = groupByOperator.getNestedPlans().get(0);
+                        if (p0.getRoots().size() != 1) {
+                            //Sort group-by currently works only for one nested plan with one root containing
+                            //an aggregate and a nested-tuple-source.
+                            continue;
+                        }
+
+                        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+                        AbstractLogicalOperator r0Logical = (AbstractLogicalOperator) r0.getValue();
+                        if (r0Logical.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+                            //we only rewrite aggregation function; do nothing for running aggregates
+                            continue;
+                        }
+                        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+                        AbstractLogicalOperator aggInputOp = (AbstractLogicalOperator) aggOp.getInputs().get(0)
+                                .getValue();
+                        if (aggInputOp.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                            continue;
+                        }
+
+                        boolean hasIntermediateAggregate = generateMergeAggregationExpressions(groupByOperator, context);
+                        if (!hasIntermediateAggregate) {
+                            continue;
+                        }
+
+                        //replace preclustered gby with sort gby
+                        op.setPhysicalOperator(new SortGroupByPOperator(groupByOperator.getGroupByList(), context
+                                .getPhysicalOptimizationConfig().getMaxFramesExternalGroupBy(), sortPhysicalOperator
+                                .getSortColumns()));
+
+                        // remove the stable sort operator
+                        op.getInputs().clear();
+                        op.getInputs().addAll(op2.getInputs());
+                        changed = true;
+                    }
+                }
+                continue;
+            } else {
+                continue;
+            }
+        }
+        return changed;
+    }
+
+    private boolean generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context)
+            throws AlgebricksException {
+        if (gby.getNestedPlans().size() != 1) {
+            throw new AlgebricksException(
+                    "External/sort group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        ILogicalPlan p0 = gby.getNestedPlans().get(0);
+        if (p0.getRoots().size() != 1) {
+            throw new AlgebricksException(
+                    "External/sort group-by currently works only for one nested plan with one root containing"
+                            + "an aggregate and a nested-tuple-source.");
+        }
+        IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
+                .getMergeAggregationExpressionFactory();
+        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+        List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
+        List<LogicalVariable> originalAggVars = aggOp.getVariables();
+        int n = aggOp.getExpressions().size();
+        List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
+        for (int i = 0; i < n; i++) {
+            ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(
+                    originalAggVars.get(i), aggFuncRefs.get(i).getValue(), context);
+            if (mergeExpr == null) {
+                return false;
+            }
+            mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
+        }
+        aggOp.setMergeExpressions(mergeExpressionRefs);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java
new file mode 100644
index 0000000..16a71a4
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushMapOperatorDownThroughProductRule.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushMapOperatorDownThroughProductRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (!op1.isMap()) {
+            return false;
+        }
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
+            return false;
+        }
+
+        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op1, used);
+
+        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+        ILogicalOperator b0 = b0Ref.getValue();
+        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
+        VariableUtilities.getLiveVariables(b0, b0Scm);
+        if (b0Scm.containsAll(used)) {
+            // push operator on left branch
+            op2Ref.setValue(b0);
+            b0Ref.setValue(op1);
+            opRef.setValue(op2);
+            return true;
+        } else {
+            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+            ILogicalOperator b1 = b1Ref.getValue();
+            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(b1, b1Scm);
+            if (b1Scm.containsAll(used)) {
+                // push operator on right branch
+                op2Ref.setValue(b1);
+                b1Ref.setValue(op1);
+                opRef.setValue(op2);
+                return true;
+            } else {
+                return false;
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
new file mode 100644
index 0000000..476096d
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushNestedOrderByUnderPreSortedGroupByRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        if (op.getPhysicalOperator() == null) {
+            return false;
+        }
+        AbstractPhysicalOperator pOp = (AbstractPhysicalOperator) op.getPhysicalOperator();
+        if (pOp.getOperatorTag() != PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
+            return false;
+        }
+        GroupByOperator gby = (GroupByOperator) op;
+        ILogicalPlan plan = gby.getNestedPlans().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) plan.getRoots().get(0).getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.ORDER) {
+            return false;
+        }
+        OrderOperator order1 = (OrderOperator) op2;
+        if (!isIndependentFromChildren(order1)) {
+            return false;
+        }
+        AbstractPhysicalOperator pOrder1 = (AbstractPhysicalOperator) op2.getPhysicalOperator();
+        if (pOrder1.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT
+                && pOrder1.getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT) {
+            return false;
+        }
+        // StableSortPOperator sort1 = (StableSortPOperator) pOrder1;
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        if (op3.getOperatorTag() != LogicalOperatorTag.ORDER) {
+            return false;
+        }
+        AbstractPhysicalOperator pOp3 = (AbstractPhysicalOperator) op3.getPhysicalOperator();
+        if (pOp3.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) {
+            return false;
+        }
+        OrderOperator order2 = (OrderOperator) op3;
+        StableSortPOperator sort2 = (StableSortPOperator) pOp3;
+        // int n1 = sort1.getSortColumns().length;
+        // int n2 = sort2.getSortColumns().length;
+        // OrderColumn[] sortColumns = new OrderColumn[n2 + n1];
+        // System.arraycopy(sort2.getSortColumns(), 0, sortColumns, 0, n2);
+        // int k = 0;
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : order1.getOrderExpressions()) {
+            order2.getOrderExpressions().add(oe);
+            // sortColumns[n2 + k] = sort1.getSortColumns()[k];
+            // ++k;
+        }
+        // sort2.setSortColumns(sortColumns);
+        sort2.computeDeliveredProperties(order2, null);
+        // remove order1
+        ILogicalOperator underOrder1 = order1.getInputs().get(0).getValue();
+        opRef2.setValue(underOrder1);
+        return true;
+    }
+
+    private boolean isIndependentFromChildren(OrderOperator order1) throws AlgebricksException {
+        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(order1, free);
+        Set<LogicalVariable> usedInOrder = new HashSet<LogicalVariable>();
+        VariableUtilities.getUsedVariables(order1, usedInOrder);
+        return free.containsAll(usedInOrder);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
new file mode 100644
index 0000000..b8ff247
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Pushes projections through its input operator, provided that operator does
+ * not produce the projected variables.
+ * 
+ * @author Nicola
+ */
+public class PushProjectDownRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+            return false;
+        }
+        ProjectOperator pi = (ProjectOperator) op;
+        Mutable<ILogicalOperator> opRef2 = pi.getInputs().get(0);
+
+        HashSet<LogicalVariable> toPush = new HashSet<LogicalVariable>();
+        toPush.addAll(pi.getVariables());
+
+        Pair<Boolean, Boolean> p = pushThroughOp(toPush, opRef2, op, context);
+        boolean smthWasPushed = p.first;
+        if (p.second) { // the original projection is redundant
+            opRef.setValue(op.getInputs().get(0).getValue());
+            smthWasPushed = true;
+        }
+
+        return smthWasPushed;
+    }
+
+    private static Pair<Boolean, Boolean> pushThroughOp(HashSet<LogicalVariable> toPush,
+            Mutable<ILogicalOperator> opRef2, ILogicalOperator initialOp, IOptimizationContext context)
+            throws AlgebricksException {
+        List<LogicalVariable> initProjectList = new ArrayList<LogicalVariable>(toPush);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        do {
+            if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
+                    || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE
+                    || op2.getOperatorTag() == LogicalOperatorTag.PROJECT
+                    || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE
+                    || op2.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
+                return new Pair<Boolean, Boolean>(false, false);
+            }
+            if (!op2.isMap()) {
+                break;
+            }
+            LinkedList<LogicalVariable> usedVars = new LinkedList<LogicalVariable>();
+            VariableUtilities.getUsedVariables(op2, usedVars);
+            toPush.addAll(usedVars);
+            LinkedList<LogicalVariable> producedVars = new LinkedList<LogicalVariable>();
+            VariableUtilities.getProducedVariables(op2, producedVars);
+            toPush.removeAll(producedVars);
+            // we assume pipelineable ops. have only one input
+            opRef2 = op2.getInputs().get(0);
+            op2 = (AbstractLogicalOperator) opRef2.getValue();
+        } while (true);
+
+        LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(op2, produced2);
+        LinkedList<LogicalVariable> used2 = new LinkedList<LogicalVariable>();
+        VariableUtilities.getUsedVariables(op2, used2);
+
+        boolean canCommuteProjection = initProjectList.containsAll(toPush) && initProjectList.containsAll(produced2)
+                && initProjectList.containsAll(used2);
+        // if true, we can get rid of the initial projection
+
+        // get rid of useless decor vars.
+        if (!canCommuteProjection && op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
+            boolean gbyChanged = false;
+            GroupByOperator gby = (GroupByOperator) op2;
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+                LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
+                if (!toPush.contains(decorVar)) {
+                    used2.remove(decorVar);
+                    gbyChanged = true;
+                } else {
+                    newDecorList.add(p);
+                }
+            }
+            gby.getDecorList().clear();
+            gby.getDecorList().addAll(newDecorList);
+            if (gbyChanged) {
+                context.computeAndSetTypeEnvironmentForOperator(gby);
+            }
+        }
+        used2.clear();
+        VariableUtilities.getUsedVariables(op2, used2);
+
+        toPush.addAll(used2); // remember that toPush is a Set
+        toPush.removeAll(produced2);
+
+        if (toPush.isEmpty()) {
+            return new Pair<Boolean, Boolean>(false, false);
+        }
+
+        boolean smthWasPushed = false;
+        for (Mutable<ILogicalOperator> c : op2.getInputs()) {
+            if (pushNeededProjections(toPush, c, context, initialOp)) {
+                smthWasPushed = true;
+            }
+        }
+        if (op2.hasNestedPlans()) {
+            AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op2;
+            for (ILogicalPlan p : n.getNestedPlans()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    if (pushNeededProjections(toPush, r, context, initialOp)) {
+                        smthWasPushed = true;
+                    }
+                }
+            }
+        }
+        return new Pair<Boolean, Boolean>(smthWasPushed, canCommuteProjection);
+    }
+
+    // It does not try to push above another Projection.
+    private static boolean pushNeededProjections(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef,
+            IOptimizationContext context, ILogicalOperator initialOp) throws AlgebricksException {
+        HashSet<LogicalVariable> allP = new HashSet<LogicalVariable>();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        VariableUtilities.getLiveVariables(op, allP);
+
+        HashSet<LogicalVariable> toProject = new HashSet<LogicalVariable>();
+        for (LogicalVariable v : toPush) {
+            if (allP.contains(v)) {
+                toProject.add(v);
+            }
+        }
+        if (toProject.equals(allP)) {
+            // projection would be redundant, since we would project everything
+            // but we can try with the children
+            boolean push = false;
+            if (pushThroughOp(toProject, opRef, initialOp, context).first) {
+                push = true;
+            }
+            return push;
+        } else {
+            return pushAllProjectionsOnTopOf(toProject, opRef, context, initialOp);
+        }
+    }
+
+    // It does not try to push above another Projection.
+    private static boolean pushAllProjectionsOnTopOf(Collection<LogicalVariable> toPush,
+            Mutable<ILogicalOperator> opRef, IOptimizationContext context, ILogicalOperator initialOp)
+            throws AlgebricksException {
+        if (toPush.isEmpty()) {
+            return false;
+        }
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+        if (context.checkAndAddToAlreadyCompared(initialOp, op)) {
+            return false;
+        }
+
+        switch (op.getOperatorTag()) {
+            case EXCHANGE: {
+                opRef = opRef.getValue().getInputs().get(0);
+                op = (AbstractLogicalOperator) opRef.getValue();
+                break;
+            }
+            case PROJECT: {
+                return false;
+            }
+        }
+
+        ProjectOperator pi2 = new ProjectOperator(new ArrayList<LogicalVariable>(toPush));
+        pi2.getInputs().add(new MutableObject<ILogicalOperator>(op));
+        opRef.setValue(pi2);
+        pi2.setExecutionMode(op.getExecutionMode());
+        context.computeAndSetTypeEnvironmentForOperator(pi2);
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
new file mode 100644
index 0000000..b17d50f
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushProjectIntoDataSourceScanRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getInputs().size() <= 0)
+            return false;
+        AbstractLogicalOperator project = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+        if (project.getOperatorTag() != LogicalOperatorTag.PROJECT)
+            return false;
+        AbstractLogicalOperator exchange = (AbstractLogicalOperator) project.getInputs().get(0).getValue();
+        if (exchange.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
+            return false;
+        AbstractLogicalOperator inputOp = (AbstractLogicalOperator) exchange.getInputs().get(0).getValue();
+        if (inputOp.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
+            return false;
+        DataSourceScanOperator scanOp = (DataSourceScanOperator) inputOp;
+        ProjectOperator projectOp = (ProjectOperator) project;
+        scanOp.addProjectVariables(projectOp.getVariables());
+        if (op.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
+            op.getInputs().set(0, project.getInputs().get(0));
+        } else {
+            op.getInputs().set(0, exchange.getInputs().get(0));
+        }
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
new file mode 100644
index 0000000..cc65996
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushSelectDownRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+        if (context.checkAndAddToAlreadyCompared(op, op2)) {
+            return false;
+        }
+
+        LogicalOperatorTag tag2 = op2.getOperatorTag();
+
+        if (tag2 == LogicalOperatorTag.INNERJOIN || tag2 == LogicalOperatorTag.LEFTOUTERJOIN
+                || tag2 == LogicalOperatorTag.REPLICATE) {
+            return false;
+        } else { // not a join
+            boolean res = propagateSelectionRec(opRef, opRef2);
+            if (res) {
+                OperatorPropertiesUtil.typeOpRec(opRef, context);
+            }
+            return res;
+        }
+    }
+
+    private static boolean propagateSelectionRec(Mutable<ILogicalOperator> sigmaRef, Mutable<ILogicalOperator> opRef2)
+            throws AlgebricksException {
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+        if (op2.getInputs().size() != 1 || op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+            return false;
+        }
+
+        SelectOperator sigma = (SelectOperator) sigmaRef.getValue();
+        LinkedList<LogicalVariable> usedInSigma = new LinkedList<LogicalVariable>();
+        sigma.getCondition().getValue().getUsedVariables(usedInSigma);
+
+        LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(op2, produced2);
+        if (OperatorPropertiesUtil.disjoint(produced2, usedInSigma)) {
+            // just swap
+            opRef2.setValue(sigma);
+            sigmaRef.setValue(op2);
+            List<Mutable<ILogicalOperator>> sigmaInpList = sigma.getInputs();
+            sigmaInpList.clear();
+            sigmaInpList.addAll(op2.getInputs());
+            List<Mutable<ILogicalOperator>> op2InpList = op2.getInputs();
+            op2InpList.clear();
+            op2InpList.add(opRef2);
+            propagateSelectionRec(opRef2, sigma.getInputs().get(0));
+            return true;
+
+        }
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
new file mode 100644
index 0000000..e29299c
--- /dev/null
+++ b/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
@@ -0,0 +1,309 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushSelectIntoJoinRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        Collection<LogicalVariable> joinLiveVarsLeft = new HashSet<LogicalVariable>();
+        Collection<LogicalVariable> joinLiveVarsRight = new HashSet<LogicalVariable>();
+        Collection<LogicalVariable> liveInOpsToPushLeft = new HashSet<LogicalVariable>();
+        Collection<LogicalVariable> liveInOpsToPushRight = new HashSet<LogicalVariable>();
+
+        List<ILogicalOperator> pushedOnLeft = new ArrayList<ILogicalOperator>();
+        List<ILogicalOperator> pushedOnRight = new ArrayList<ILogicalOperator>();
+        LinkedList<ILogicalOperator> notPushedStack = new LinkedList<ILogicalOperator>();
+        Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
+        Collection<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
+
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+        SelectOperator select = (SelectOperator) op;
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getValue();
+        AbstractLogicalOperator op2 = son;
+        boolean needToPushOps = false;
+        while (son.isMap()) {
+            needToPushOps = true;
+            Mutable<ILogicalOperator> opRefLink = son.getInputs().get(0);
+            son = (AbstractLogicalOperator) opRefLink.getValue();
+        }
+
+        if (son.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+                && son.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+            return false;
+        }
+        boolean isLoj = son.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN;
+        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) son;
+
+        Mutable<ILogicalOperator> joinBranchLeftRef = join.getInputs().get(0);
+        Mutable<ILogicalOperator> joinBranchRightRef = join.getInputs().get(1);
+
+        if (needToPushOps) {
+            ILogicalOperator joinBranchLeft = joinBranchLeftRef.getValue();
+            ILogicalOperator joinBranchRight = joinBranchRightRef.getValue();
+            VariableUtilities.getLiveVariables(joinBranchLeft, joinLiveVarsLeft);
+            VariableUtilities.getLiveVariables(joinBranchRight, joinLiveVarsRight);
+            Mutable<ILogicalOperator> opIterRef = opRef2;
+            ILogicalOperator opIter = op2;
+            while (opIter != join) {
+                LogicalOperatorTag tag = ((AbstractLogicalOperator) opIter).getOperatorTag();
+                if (tag == LogicalOperatorTag.PROJECT) {
+                    notPushedStack.addFirst(opIter);
+                } else {
+                    VariableUtilities.getUsedVariables(opIter, usedVars);
+                    VariableUtilities.getProducedVariables(opIter, producedVars);
+                    if (joinLiveVarsLeft.containsAll(usedVars)) {
+                        pushedOnLeft.add(opIter);
+                        liveInOpsToPushLeft.addAll(producedVars);
+                    } else if (joinLiveVarsRight.containsAll(usedVars)) {
+                        pushedOnRight.add(opIter);
+                        liveInOpsToPushRight.addAll(producedVars);
+                    } else {
+                        return false;
+                    }
+                }
+                opIterRef = opIter.getInputs().get(0);
+                opIter = opIterRef.getValue();
+            }
+            if (isLoj && pushedOnLeft.isEmpty()) {
+                return false;
+            }
+        }
+
+        boolean intersectsAllBranches = true;
+        boolean[] intersectsBranch = new boolean[join.getInputs().size()];
+        LinkedList<LogicalVariable> selectVars = new LinkedList<LogicalVariable>();
+        select.getCondition().getValue().getUsedVariables(selectVars);
+        int i = 0;
+        for (Mutable<ILogicalOperator> branch : join.getInputs()) {
+            LinkedList<LogicalVariable> branchVars = new LinkedList<LogicalVariable>();
+            VariableUtilities.getLiveVariables(branch.getValue(), branchVars);
+            if (i == 0) {
+                branchVars.addAll(liveInOpsToPushLeft);
+            } else {
+                branchVars.addAll(liveInOpsToPushRight);
+            }
+            if (OperatorPropertiesUtil.disjoint(selectVars, branchVars)) {
+                intersectsAllBranches = false;
+            } else {
+                intersectsBranch[i] = true;
+            }
+            i++;
+        }
+        if (!intersectsBranch[0] && !intersectsBranch[1]) {
+            return false;
+        }
+        if (needToPushOps) {
+            pushOps(pushedOnLeft, joinBranchLeftRef, context);
+            pushOps(pushedOnRight, joinBranchRightRef, context);
+        }
+        if (intersectsAllBranches) {
+            addCondToJoin(select, join, context);
+        } else { // push down
+            Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator();
+            ILogicalExpression selectCondition = select.getCondition().getValue();
+            boolean lojToInner = false;
+            for (int j = 0; j < intersectsBranch.length; j++) {
+                Mutable<ILogicalOperator> branch = branchIter.next();
+                boolean inter = intersectsBranch[j];
+                if (inter) {
+                    if (j > 0 && isLoj) {
+                        // if a left outer join, if the select condition is not-null filtering,
+                        // we rewrite left outer join
+                        // to inner join for this case.
+                        if (containsNotNullFiltering(selectCondition)) {
+                            lojToInner = true;
+                        }
+                    }
+                    if ((j > 0 && isLoj) && containsNullFiltering(selectCondition)) {
+                        // Select is-null($$var) cannot be pushed in the right branch of a LOJ;
+                        notPushedStack.addFirst(select);
+                    } else {
+                        // Conditions for the left branch can always be pushed.
+                        // Other conditions can be pushed to the right branch of a LOJ.
+                        copySelectToBranch(select, branch, context);
+                    }
+                }
+            }
+            if (lojToInner) {
+                // Rewrites left outer join  to inner join.
+                InnerJoinOperator innerJoin = new InnerJoinOperator(join.getCondition());
+                innerJoin.getInputs().addAll(join.getInputs());
+                join = innerJoin;
+                context.computeAndSetTypeEnvironmentForOperator(join);
+            }
+        }
+        ILogicalOperator top = join;
+        for (ILogicalOperator npOp : notPushedStack) {
+            List<Mutable<ILogicalOperator>> npInpList = npOp.getInputs();
+            npInpList.clear();
+            npInpList.add(new MutableObject<ILogicalOperator>(top));
+            context.computeAndSetTypeEnvironmentForOperator(npOp);
+            top = npOp;
+        }
+        opRef.setValue(top);
+        return true;
+
+    }
+
+    private void pushOps(List<ILogicalOperator> opList, Mutable<ILogicalOperator> joinBranch,
+            IOptimizationContext context) throws AlgebricksException {
+        ILogicalOperator topOp = joinBranch.getValue();
+        ListIterator<ILogicalOperator> iter = opList.listIterator(opList.size());
+        while (iter.hasPrevious()) {
+            ILogicalOperator op = iter.previous();
+            List<Mutable<ILogicalOperator>> opInpList = op.getInputs();
+            opInpList.clear();
+            opInpList.add(new MutableObject<ILogicalOperator>(topOp));
+            topOp = op;
+            context.computeAndSetTypeEnvironmentForOperator(op);
+        }
+        joinBranch.setValue(topOp);
+    }
+
+    private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join,
+            IOptimizationContext context) {
+        ILogicalExpression cond = join.getCondition().getValue();
+        if (OperatorPropertiesUtil.isAlwaysTrueCond(cond)) { // the join was a product
+            join.getCondition().setValue(select.getCondition().getValue());
+        } else {
+            boolean bAddedToConj = false;
+            if (cond.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                AbstractFunctionCallExpression fcond = (AbstractFunctionCallExpression) cond;
+                if (fcond.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND)) {
+                    AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(context
+                            .getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND));
+                    newCond.getArguments().add(select.getCondition());
+                    newCond.getArguments().addAll(fcond.getArguments());
+                    join.getCondition().setValue(newCond);
+                    bAddedToConj = true;
+                }
+            }
+            if (!bAddedToConj) {
+                AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(context.getMetadataProvider()
+                        .lookupFunction(AlgebricksBuiltinFunctions.AND), select.getCondition(),
+                        new MutableObject<ILogicalExpression>(join.getCondition().getValue()));
+                join.getCondition().setValue(newCond);
+            }
+        }
+    }
+
+    private static void copySelectToBranch(SelectOperator select, Mutable<ILogicalOperator> branch,
+            IOptimizationContext context) throws AlgebricksException {
+        ILogicalOperator newSelect = new SelectOperator(select.getCondition(), select.getRetainNull(),
+                select.getNullPlaceholderVariable());
+        Mutable<ILogicalOperator> newRef = new MutableObject<ILogicalOperator>(branch.getValue());
+        newSelect.getInputs().add(newRef);
+        branch.setValue(newSelect);
+        context.computeAndSetTypeEnvironmentForOperator(newSelect);
+    }
+
+    /**
+     * Whether the expression contains a not-null filtering
+     * 
+     * @param expr
+     * @return true if the expression contains a not-null filtering function call; false otherwise.
+     */
+    private boolean containsNotNullFiltering(ILogicalExpression expr) {
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        ScalarFunctionCallExpression func = (ScalarFunctionCallExpression) expr;
+        if (func.getFunctionIdentifier() == AlgebricksBuiltinFunctions.AND) {
+            for (Mutable<ILogicalExpression> argumentRef : func.getArguments()) {
+                if (containsNotNullFiltering(argumentRef.getValue())) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        if (func.getFunctionIdentifier() != AlgebricksBuiltinFunctions.NOT) {
+            return false;
+        }
+        ILogicalExpression arg = func.getArguments().get(0).getValue();
+        if (arg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        ScalarFunctionCallExpression func2 = (ScalarFunctionCallExpression) arg;
+        if (func2.getFunctionIdentifier() != AlgebricksBuiltinFunctions.IS_NULL) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Whether the expression contains a null filtering
+     * 
+     * @param expr
+     * @return true if the expression contains a null filtering function call; false otherwise.
+     */
+    private boolean containsNullFiltering(ILogicalExpression expr) {
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        ScalarFunctionCallExpression func = (ScalarFunctionCallExpression) expr;
+        if (func.getFunctionIdentifier() == AlgebricksBuiltinFunctions.AND) {
+            for (Mutable<ILogicalExpression> argumentRef : func.getArguments()) {
+                if (containsNullFiltering(argumentRef.getValue())) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        if (func.getFunctionIdentifier() != AlgebricksBuiltinFunctions.IS_NULL) {
+            return false;
+        }
+        return true;
+    }
+}
\ No newline at end of file

[51/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

Change folder structure for Java repackage

Change only the folders, not the files, for our package name change.
This will break the build, and needs to be followed by a change to
the package name in all of the source files. However performing
the folder move and file change in two steps lets Git understand
that the files are the same, and lets us track revisions across
those files.

Change-Id: I08aff75e25ac7c6298c32cf3402febbc4a318c2a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/307
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Chris Hillery <ce...@lambda.nu>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/commit/9939b48e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/tree/9939b48e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/diff/9939b48e

Branch: refs/heads/master
Commit: 9939b48ecc4623145383e847cc6adbd4b800c609
Parents: 4db4f57
Author: Ian Maxon <im...@apache.org>
Authored: Thu Aug 20 12:14:49 2015 -0700
Committer: Ian Maxon <im...@apache.org>
Committed: Sat Aug 22 12:56:08 2015 -0700

----------------------------------------------------------------------
 .../AlgebricksAbsolutePartitionConstraint.java  |   32 -
 .../AlgebricksCountPartitionConstraint.java     |   34 -
 .../AlgebricksPartitionConstraint.java          |   24 -
 .../AlgebricksPartitionConstraintHelper.java    |   58 -
 .../common/exceptions/AlgebricksException.java  |   34 -
 .../exceptions/NotImplementedException.java     |   35 -
 .../algebricks/common/utils/ListSet.java        |  125 --
 .../hyracks/algebricks/common/utils/Pair.java   |   50 -
 .../hyracks/algebricks/common/utils/Triple.java |   46 -
 .../AlgebricksAbsolutePartitionConstraint.java  |   32 +
 .../AlgebricksCountPartitionConstraint.java     |   34 +
 .../AlgebricksPartitionConstraint.java          |   24 +
 .../AlgebricksPartitionConstraintHelper.java    |   58 +
 .../common/exceptions/AlgebricksException.java  |   34 +
 .../exceptions/NotImplementedException.java     |   35 +
 .../algebricks/common/utils/ListSet.java        |  125 ++
 .../hyracks/algebricks/common/utils/Pair.java   |   50 +
 .../hyracks/algebricks/common/utils/Triple.java |   46 +
 .../api/AbstractCompilerFactoryBuilder.java     |  230 ---
 .../api/HeuristicCompilerFactoryBuilder.java    |  107 --
 .../algebricks/compiler/api/ICompiler.java      |   26 -
 .../compiler/api/ICompilerFactory.java          |   22 -
 .../PrioritizedRuleController.java              |   63 -
 .../SequentialFixpointRuleController.java       |   59 -
 .../SequentialOnceRuleController.java           |   46 -
 .../api/AbstractCompilerFactoryBuilder.java     |  230 +++
 .../api/HeuristicCompilerFactoryBuilder.java    |  107 ++
 .../algebricks/compiler/api/ICompiler.java      |   26 +
 .../compiler/api/ICompilerFactory.java          |   22 +
 .../PrioritizedRuleController.java              |   63 +
 .../SequentialFixpointRuleController.java       |   59 +
 .../SequentialOnceRuleController.java           |   46 +
 .../algebricks/core/algebra/base/Counter.java   |   38 -
 .../core/algebra/base/EquivalenceClass.java     |  143 --
 .../core/algebra/base/IHyracksJobBuilder.java   |   58 -
 .../core/algebra/base/ILogicalExpression.java   |   72 -
 .../core/algebra/base/ILogicalOperator.java     |  102 --
 .../core/algebra/base/ILogicalPlan.java         |   23 -
 .../algebra/base/ILogicalPlanAndMetadata.java   |   26 -
 .../core/algebra/base/IOptimizationContext.java |   90 --
 .../core/algebra/base/IPhysicalOperator.java    |   75 -
 .../core/algebra/base/LogicalExpressionTag.java |   19 -
 .../core/algebra/base/LogicalOperatorTag.java   |   51 -
 .../core/algebra/base/LogicalVariable.java      |   51 -
 .../core/algebra/base/OperatorAnnotations.java  |   27 -
 .../core/algebra/base/PhysicalOperatorTag.java  |   72 -
 .../AbstractFunctionCallExpression.java         |  355 -----
 .../expressions/AbstractLogicalExpression.java  |   44 -
 .../AggregateFunctionCallExpression.java        |  102 --
 .../BroadcastExpressionAnnotation.java          |   45 -
 .../algebra/expressions/ConstantExpression.java |  174 ---
 .../ExpressionAnnotationNoCopyImpl.java         |   36 -
 .../expressions/IAlgebricksConstantValue.java   |   23 -
 .../expressions/IExpressionAnnotation.java      |   23 -
 .../IExpressionEvalSizeComputer.java            |   23 -
 .../expressions/IExpressionRuntimeProvider.java |   46 -
 .../expressions/IExpressionTypeComputer.java    |   24 -
 .../expressions/ILogicalExpressionJobGen.java   |   48 -
 .../IMergeAggregationExpressionFactory.java     |   25 -
 .../expressions/INullableTypeComputer.java      |   25 -
 .../IPartialAggregationTypeComputer.java        |   24 -
 .../IVariableEvalSizeEnvironment.java           |   24 -
 .../expressions/IVariableTypeEnvironment.java   |   34 -
 .../IndexedNLJoinExpressionAnnotation.java      |   40 -
 ...obGenToExpressionRuntimeProviderAdapter.java |  214 ---
 .../ScalarFunctionCallExpression.java           |   54 -
 .../StatefulFunctionCallExpression.java         |   65 -
 .../UnnestingFunctionCallExpression.java        |   65 -
 .../VariableReferenceExpression.java            |  110 --
 .../algebra/functions/AbstractFunctionInfo.java |   35 -
 .../functions/AlgebricksBuiltinFunctions.java   |   69 -
 .../algebra/functions/FunctionIdentifier.java   |   70 -
 .../core/algebra/functions/IFunctionInfo.java   |   21 -
 .../core/algebra/metadata/IDataSink.java        |   25 -
 .../core/algebra/metadata/IDataSource.java      |   30 -
 .../core/algebra/metadata/IDataSourceIndex.java |   21 -
 .../metadata/IDataSourcePropertiesProvider.java |   24 -
 .../algebra/metadata/IMetadataProvider.java     |  192 ---
 .../logical/AbstractAssignOperator.java         |   70 -
 .../logical/AbstractBinaryJoinOperator.java     |   78 --
 .../logical/AbstractDataSourceOperator.java     |   35 -
 .../AbstractExtensibleLogicalOperator.java      |   56 -
 .../logical/AbstractLogicalOperator.java        |  190 ---
 .../AbstractOperatorWithNestedPlans.java        |   90 --
 .../operators/logical/AbstractScanOperator.java |   63 -
 .../logical/AbstractUnnestOperator.java         |   49 -
 .../operators/logical/AggregateOperator.java    |  109 --
 .../operators/logical/AssignOperator.java       |  119 --
 .../logical/DataSourceScanOperator.java         |  133 --
 .../operators/logical/DistinctOperator.java     |  134 --
 .../logical/DistributeResultOperator.java       |   93 --
 .../logical/EmptyTupleSourceOperator.java       |   96 --
 .../operators/logical/ExchangeOperator.java     |   76 -
 .../operators/logical/ExtensionOperator.java    |  120 --
 .../logical/ExternalDataLookupOperator.java     |  115 --
 .../operators/logical/GroupByOperator.java      |  284 ----
 .../operators/logical/IOperatorExtension.java   |   50 -
 .../operators/logical/IOperatorSchema.java      |   33 -
 .../logical/IndexInsertDeleteOperator.java      |  142 --
 .../operators/logical/InnerJoinOperator.java    |   53 -
 .../operators/logical/InsertDeleteOperator.java |  129 --
 .../logical/LeftOuterJoinOperator.java          |   72 -
 .../operators/logical/LimitOperator.java        |  112 --
 .../operators/logical/MaterializeOperator.java  |   66 -
 .../logical/NestedTupleSourceOperator.java      |  108 --
 .../operators/logical/OrderOperator.java        |  159 ---
 .../logical/PartitioningSplitOperator.java      |  115 --
 .../operators/logical/ProjectOperator.java      |   83 --
 .../operators/logical/ReplicateOperator.java    |  117 --
 .../logical/RunningAggregateOperator.java       |   79 --
 .../operators/logical/ScriptOperator.java       |  106 --
 .../operators/logical/SelectOperator.java       |  121 --
 .../algebra/operators/logical/SinkOperator.java |   86 --
 .../operators/logical/SubplanOperator.java      |  111 --
 .../operators/logical/TokenizeOperator.java     |  190 ---
 .../operators/logical/UnionAllOperator.java     |  120 --
 .../operators/logical/UnnestMapOperator.java    |  134 --
 .../operators/logical/UnnestOperator.java       |  144 --
 .../operators/logical/UpdateOperator.java       |   69 -
 .../operators/logical/WriteOperator.java        |   93 --
 .../operators/logical/WriteResultOperator.java  |  109 --
 .../visitors/FDsAndEquivClassesVisitor.java     |  763 ----------
 .../visitors/IsomorphismOperatorVisitor.java    |  575 --------
 .../logical/visitors/IsomorphismUtilities.java  |   70 -
 .../IsomorphismVariableMappingVisitor.java      |  447 ------
 .../visitors/LogicalPropertiesVisitor.java      |  350 -----
 .../visitors/OperatorDeepCopyVisitor.java       |  368 -----
 .../visitors/ProducedVariableVisitor.java       |  275 ----
 .../logical/visitors/SchemaVariableVisitor.java |  330 -----
 .../visitors/SubstituteVariableVisitor.java     |  469 -------
 .../logical/visitors/UsedVariableVisitor.java   |  407 ------
 .../logical/visitors/VariableUtilities.java     |   97 --
 .../physical/AbstractExchangePOperator.java     |   51 -
 .../physical/AbstractHashJoinPOperator.java     |  191 ---
 .../physical/AbstractJoinPOperator.java         |   54 -
 .../physical/AbstractPhysicalOperator.java      |  157 ---
 .../AbstractPreclusteredGroupByPOperator.java   |  265 ----
 ...gatePropertiesForUsedVariablesPOperator.java |   44 -
 .../physical/AbstractScanPOperator.java         |   33 -
 .../physical/AbstractStableSortPOperator.java   |  123 --
 .../operators/physical/AggregatePOperator.java  |  121 --
 .../operators/physical/AssignPOperator.java     |  123 --
 .../operators/physical/BroadcastPOperator.java  |   70 -
 .../operators/physical/BulkloadPOperator.java   |  120 --
 .../physical/DataSourceScanPOperator.java       |   94 --
 .../physical/DistributeResultPOperator.java     |  114 --
 .../physical/EmptyTupleSourcePOperator.java     |   72 -
 .../physical/ExternalGroupByPOperator.java      |  271 ----
 .../HashPartitionExchangePOperator.java         |  102 --
 .../HashPartitionMergeExchangePOperator.java    |  167 ---
 .../physical/HybridHashJoinPOperator.java       |  281 ----
 .../physical/InMemoryHashJoinPOperator.java     |  139 --
 .../physical/InMemoryStableSortPOperator.java   |   82 --
 .../physical/IndexBulkloadPOperator.java        |  140 --
 .../physical/IndexInsertDeletePOperator.java    |  131 --
 .../physical/InsertDeletePOperator.java         |  121 --
 .../physical/MaterializePOperator.java          |   94 --
 .../MicroPreclusteredGroupByPOperator.java      |   94 --
 .../operators/physical/NLJoinPOperator.java     |  288 ----
 .../physical/NestedTupleSourcePOperator.java    |  107 --
 .../physical/OneToOneExchangePOperator.java     |   61 -
 .../physical/PreSortedDistinctByPOperator.java  |  145 --
 .../physical/PreclusteredGroupByPOperator.java  |  104 --
 .../physical/RandomMergeExchangePOperator.java  |   60 -
 .../physical/RandomPartitionPOperator.java      |   86 --
 .../physical/RangePartitionMergePOperator.java  |  151 --
 .../physical/RangePartitionPOperator.java       |  122 --
 .../operators/physical/ReplicatePOperator.java  |   95 --
 .../physical/RunningAggregatePOperator.java     |  104 --
 .../operators/physical/SinkPOperator.java       |   97 --
 .../operators/physical/SinkWritePOperator.java  |  111 --
 .../physical/SortGroupByPOperator.java          |  291 ----
 .../physical/SortMergeExchangePOperator.java    |  156 ---
 .../operators/physical/StableSortPOperator.java |   95 --
 .../physical/StreamLimitPOperator.java          |  107 --
 .../physical/StreamProjectPOperator.java        |   88 --
 .../physical/StreamSelectPOperator.java         |   80 --
 .../StringStreamingScriptPOperator.java         |   82 --
 .../operators/physical/SubplanPOperator.java    |  111 --
 .../operators/physical/TokenizePOperator.java   |  108 --
 .../operators/physical/UnionAllPOperator.java   |  105 --
 .../operators/physical/UnnestPOperator.java     |   97 --
 .../physical/WriteResultPOperator.java          |  111 --
 .../core/algebra/plan/ALogicalPlanImpl.java     |   52 -
 .../LogicalExpressionPrettyPrintVisitor.java    |   70 -
 .../LogicalOperatorPrettyPrintVisitor.java      |  431 ------
 .../core/algebra/prettyprint/PlanPlotter.java   |  139 --
 .../algebra/prettyprint/PlanPrettyPrinter.java  |   90 --
 .../properties/AbstractGroupingProperty.java    |   86 --
 .../BroadcastPartitioningProperty.java          |   56 -
 .../properties/DefaultNodeGroupDomain.java      |   39 -
 .../algebra/properties/FileSplitDomain.java     |   72 -
 .../FilteredVariablePropagationPolicy.java      |   37 -
 .../properties/FunctionalDependency.java        |   53 -
 .../properties/ILocalStructuralProperty.java    |   50 -
 .../properties/ILogicalPropertiesVector.java    |   21 -
 .../core/algebra/properties/INodeDomain.java    |   22 -
 .../properties/IPartitioningProperty.java       |   81 --
 .../IPartitioningRequirementsCoordinator.java   |   98 --
 .../properties/IPhysicalPropertiesVector.java   |   44 -
 .../algebra/properties/IPropertiesComputer.java |   23 -
 .../algebra/properties/IStructuralProperty.java |   23 -
 .../properties/LocalGroupingProperty.java       |  103 --
 .../algebra/properties/LocalOrderProperty.java  |  210 ---
 .../properties/LogicalPropertiesVectorImpl.java |   43 -
 .../core/algebra/properties/OrderColumn.java    |   60 -
 .../properties/OrderedPartitionedProperty.java  |   80 --
 .../properties/PhysicalRequirements.java        |   34 -
 .../core/algebra/properties/PropertiesUtil.java |  320 -----
 .../properties/RandomPartitioningProperty.java  |   61 -
 .../algebra/properties/ResultSetDomain.java     |   27 -
 .../properties/StructuralPropertiesVector.java  |  101 --
 .../properties/TypePropagationPolicy.java       |   97 --
 .../UnorderedPartitionedProperty.java           |   64 -
 .../UnpartitionedPropertyComputer.java          |   31 -
 .../properties/VariablePropagationPolicy.java   |   78 --
 .../algebra/scripting/IScriptDescription.java   |   30 -
 .../StringStreamingScriptDescription.java       |   66 -
 .../algebra/typing/AbstractTypeEnvironment.java |   59 -
 .../core/algebra/typing/ITypeEnvPointer.java    |   21 -
 .../core/algebra/typing/ITypingContext.java     |   33 -
 .../typing/NonPropagatingTypeEnvironment.java   |   42 -
 .../algebra/typing/OpRefTypeEnvPointer.java     |   42 -
 .../PropagateOperatorInputsTypeEnvironment.java |   76 -
 .../typing/PropagatingTypeEnvironment.java      |   87 --
 .../algebra/util/OperatorManipulationUtil.java  |  225 ---
 .../algebra/util/OperatorPropertiesUtil.java    |  300 ----
 .../visitors/AbstractConstVarFunVisitor.java    |   46 -
 .../ILogicalExpressionReferenceTransform.java   |   25 -
 .../visitors/ILogicalExpressionVisitor.java     |   39 -
 .../visitors/ILogicalOperatorVisitor.java       |  117 --
 .../core/config/AlgebricksConfig.java           |   23 -
 .../algebricks/core/config/SysoutFormatter.java |   35 -
 .../impl/ConnectorPolicyAssignmentPolicy.java   |   38 -
 .../algebricks/core/jobgen/impl/JobBuilder.java |  330 -----
 .../core/jobgen/impl/JobGenContext.java         |  208 ---
 .../core/jobgen/impl/JobGenHelper.java          |  182 ---
 .../core/jobgen/impl/OperatorSchemaImpl.java    |   96 --
 .../core/jobgen/impl/PlanCompiler.java          |  125 --
 .../rewriter/base/AbstractRuleController.java   |  130 --
 .../base/AlgebricksOptimizationContext.java     |  299 ----
 .../core/rewriter/base/HeuristicOptimizer.java  |  125 --
 .../rewriter/base/IAlgebraicRewriteRule.java    |   28 -
 .../base/IOptimizationContextFactory.java       |   29 -
 .../base/PhysicalOptimizationConfig.java        |  156 ---
 .../algebricks/core/utils/Substitution.java     |   25 -
 .../algebricks/core/algebra/base/Counter.java   |   38 +
 .../core/algebra/base/EquivalenceClass.java     |  143 ++
 .../core/algebra/base/IHyracksJobBuilder.java   |   58 +
 .../core/algebra/base/ILogicalExpression.java   |   72 +
 .../core/algebra/base/ILogicalOperator.java     |  102 ++
 .../core/algebra/base/ILogicalPlan.java         |   23 +
 .../algebra/base/ILogicalPlanAndMetadata.java   |   26 +
 .../core/algebra/base/IOptimizationContext.java |   90 ++
 .../core/algebra/base/IPhysicalOperator.java    |   75 +
 .../core/algebra/base/LogicalExpressionTag.java |   19 +
 .../core/algebra/base/LogicalOperatorTag.java   |   51 +
 .../core/algebra/base/LogicalVariable.java      |   51 +
 .../core/algebra/base/OperatorAnnotations.java  |   27 +
 .../core/algebra/base/PhysicalOperatorTag.java  |   72 +
 .../AbstractFunctionCallExpression.java         |  355 +++++
 .../expressions/AbstractLogicalExpression.java  |   44 +
 .../AggregateFunctionCallExpression.java        |  102 ++
 .../BroadcastExpressionAnnotation.java          |   45 +
 .../algebra/expressions/ConstantExpression.java |  174 +++
 .../ExpressionAnnotationNoCopyImpl.java         |   36 +
 .../expressions/IAlgebricksConstantValue.java   |   23 +
 .../expressions/IExpressionAnnotation.java      |   23 +
 .../IExpressionEvalSizeComputer.java            |   23 +
 .../expressions/IExpressionRuntimeProvider.java |   46 +
 .../expressions/IExpressionTypeComputer.java    |   24 +
 .../expressions/ILogicalExpressionJobGen.java   |   48 +
 .../IMergeAggregationExpressionFactory.java     |   25 +
 .../expressions/INullableTypeComputer.java      |   25 +
 .../IPartialAggregationTypeComputer.java        |   24 +
 .../IVariableEvalSizeEnvironment.java           |   24 +
 .../expressions/IVariableTypeEnvironment.java   |   34 +
 .../IndexedNLJoinExpressionAnnotation.java      |   40 +
 ...obGenToExpressionRuntimeProviderAdapter.java |  214 +++
 .../ScalarFunctionCallExpression.java           |   54 +
 .../StatefulFunctionCallExpression.java         |   65 +
 .../UnnestingFunctionCallExpression.java        |   65 +
 .../VariableReferenceExpression.java            |  110 ++
 .../algebra/functions/AbstractFunctionInfo.java |   35 +
 .../functions/AlgebricksBuiltinFunctions.java   |   69 +
 .../algebra/functions/FunctionIdentifier.java   |   70 +
 .../core/algebra/functions/IFunctionInfo.java   |   21 +
 .../core/algebra/metadata/IDataSink.java        |   25 +
 .../core/algebra/metadata/IDataSource.java      |   30 +
 .../core/algebra/metadata/IDataSourceIndex.java |   21 +
 .../metadata/IDataSourcePropertiesProvider.java |   24 +
 .../algebra/metadata/IMetadataProvider.java     |  192 +++
 .../logical/AbstractAssignOperator.java         |   70 +
 .../logical/AbstractBinaryJoinOperator.java     |   78 ++
 .../logical/AbstractDataSourceOperator.java     |   35 +
 .../AbstractExtensibleLogicalOperator.java      |   56 +
 .../logical/AbstractLogicalOperator.java        |  190 +++
 .../AbstractOperatorWithNestedPlans.java        |   90 ++
 .../operators/logical/AbstractScanOperator.java |   63 +
 .../logical/AbstractUnnestOperator.java         |   49 +
 .../operators/logical/AggregateOperator.java    |  109 ++
 .../operators/logical/AssignOperator.java       |  119 ++
 .../logical/DataSourceScanOperator.java         |  133 ++
 .../operators/logical/DistinctOperator.java     |  134 ++
 .../logical/DistributeResultOperator.java       |   93 ++
 .../logical/EmptyTupleSourceOperator.java       |   96 ++
 .../operators/logical/ExchangeOperator.java     |   76 +
 .../operators/logical/ExtensionOperator.java    |  120 ++
 .../logical/ExternalDataLookupOperator.java     |  115 ++
 .../operators/logical/GroupByOperator.java      |  284 ++++
 .../operators/logical/IOperatorExtension.java   |   50 +
 .../operators/logical/IOperatorSchema.java      |   33 +
 .../logical/IndexInsertDeleteOperator.java      |  142 ++
 .../operators/logical/InnerJoinOperator.java    |   53 +
 .../operators/logical/InsertDeleteOperator.java |  129 ++
 .../logical/LeftOuterJoinOperator.java          |   72 +
 .../operators/logical/LimitOperator.java        |  112 ++
 .../operators/logical/MaterializeOperator.java  |   66 +
 .../logical/NestedTupleSourceOperator.java      |  108 ++
 .../operators/logical/OrderOperator.java        |  159 +++
 .../logical/PartitioningSplitOperator.java      |  115 ++
 .../operators/logical/ProjectOperator.java      |   83 ++
 .../operators/logical/ReplicateOperator.java    |  117 ++
 .../logical/RunningAggregateOperator.java       |   79 ++
 .../operators/logical/ScriptOperator.java       |  106 ++
 .../operators/logical/SelectOperator.java       |  121 ++
 .../algebra/operators/logical/SinkOperator.java |   86 ++
 .../operators/logical/SubplanOperator.java      |  111 ++
 .../operators/logical/TokenizeOperator.java     |  190 +++
 .../operators/logical/UnionAllOperator.java     |  120 ++
 .../operators/logical/UnnestMapOperator.java    |  134 ++
 .../operators/logical/UnnestOperator.java       |  144 ++
 .../operators/logical/UpdateOperator.java       |   69 +
 .../operators/logical/WriteOperator.java        |   93 ++
 .../operators/logical/WriteResultOperator.java  |  109 ++
 .../visitors/FDsAndEquivClassesVisitor.java     |  763 ++++++++++
 .../visitors/IsomorphismOperatorVisitor.java    |  575 ++++++++
 .../logical/visitors/IsomorphismUtilities.java  |   70 +
 .../IsomorphismVariableMappingVisitor.java      |  447 ++++++
 .../visitors/LogicalPropertiesVisitor.java      |  350 +++++
 .../visitors/OperatorDeepCopyVisitor.java       |  368 +++++
 .../visitors/ProducedVariableVisitor.java       |  275 ++++
 .../logical/visitors/SchemaVariableVisitor.java |  330 +++++
 .../visitors/SubstituteVariableVisitor.java     |  469 +++++++
 .../logical/visitors/UsedVariableVisitor.java   |  407 ++++++
 .../logical/visitors/VariableUtilities.java     |   97 ++
 .../physical/AbstractExchangePOperator.java     |   51 +
 .../physical/AbstractHashJoinPOperator.java     |  191 +++
 .../physical/AbstractJoinPOperator.java         |   54 +
 .../physical/AbstractPhysicalOperator.java      |  157 +++
 .../AbstractPreclusteredGroupByPOperator.java   |  265 ++++
 ...gatePropertiesForUsedVariablesPOperator.java |   44 +
 .../physical/AbstractScanPOperator.java         |   33 +
 .../physical/AbstractStableSortPOperator.java   |  123 ++
 .../operators/physical/AggregatePOperator.java  |  121 ++
 .../operators/physical/AssignPOperator.java     |  123 ++
 .../operators/physical/BroadcastPOperator.java  |   70 +
 .../operators/physical/BulkloadPOperator.java   |  120 ++
 .../physical/DataSourceScanPOperator.java       |   94 ++
 .../physical/DistributeResultPOperator.java     |  114 ++
 .../physical/EmptyTupleSourcePOperator.java     |   72 +
 .../physical/ExternalGroupByPOperator.java      |  271 ++++
 .../HashPartitionExchangePOperator.java         |  102 ++
 .../HashPartitionMergeExchangePOperator.java    |  167 +++
 .../physical/HybridHashJoinPOperator.java       |  281 ++++
 .../physical/InMemoryHashJoinPOperator.java     |  139 ++
 .../physical/InMemoryStableSortPOperator.java   |   82 ++
 .../physical/IndexBulkloadPOperator.java        |  140 ++
 .../physical/IndexInsertDeletePOperator.java    |  131 ++
 .../physical/InsertDeletePOperator.java         |  121 ++
 .../physical/MaterializePOperator.java          |   94 ++
 .../MicroPreclusteredGroupByPOperator.java      |   94 ++
 .../operators/physical/NLJoinPOperator.java     |  288 ++++
 .../physical/NestedTupleSourcePOperator.java    |  107 ++
 .../physical/OneToOneExchangePOperator.java     |   61 +
 .../physical/PreSortedDistinctByPOperator.java  |  145 ++
 .../physical/PreclusteredGroupByPOperator.java  |  104 ++
 .../physical/RandomMergeExchangePOperator.java  |   60 +
 .../physical/RandomPartitionPOperator.java      |   86 ++
 .../physical/RangePartitionMergePOperator.java  |  151 ++
 .../physical/RangePartitionPOperator.java       |  122 ++
 .../operators/physical/ReplicatePOperator.java  |   95 ++
 .../physical/RunningAggregatePOperator.java     |  104 ++
 .../operators/physical/SinkPOperator.java       |   97 ++
 .../operators/physical/SinkWritePOperator.java  |  111 ++
 .../physical/SortGroupByPOperator.java          |  291 ++++
 .../physical/SortMergeExchangePOperator.java    |  156 +++
 .../operators/physical/StableSortPOperator.java |   95 ++
 .../physical/StreamLimitPOperator.java          |  107 ++
 .../physical/StreamProjectPOperator.java        |   88 ++
 .../physical/StreamSelectPOperator.java         |   80 ++
 .../StringStreamingScriptPOperator.java         |   82 ++
 .../operators/physical/SubplanPOperator.java    |  111 ++
 .../operators/physical/TokenizePOperator.java   |  108 ++
 .../operators/physical/UnionAllPOperator.java   |  105 ++
 .../operators/physical/UnnestPOperator.java     |   97 ++
 .../physical/WriteResultPOperator.java          |  111 ++
 .../core/algebra/plan/ALogicalPlanImpl.java     |   52 +
 .../LogicalExpressionPrettyPrintVisitor.java    |   70 +
 .../LogicalOperatorPrettyPrintVisitor.java      |  431 ++++++
 .../core/algebra/prettyprint/PlanPlotter.java   |  139 ++
 .../algebra/prettyprint/PlanPrettyPrinter.java  |   90 ++
 .../properties/AbstractGroupingProperty.java    |   86 ++
 .../BroadcastPartitioningProperty.java          |   56 +
 .../properties/DefaultNodeGroupDomain.java      |   39 +
 .../algebra/properties/FileSplitDomain.java     |   72 +
 .../FilteredVariablePropagationPolicy.java      |   37 +
 .../properties/FunctionalDependency.java        |   53 +
 .../properties/ILocalStructuralProperty.java    |   50 +
 .../properties/ILogicalPropertiesVector.java    |   21 +
 .../core/algebra/properties/INodeDomain.java    |   22 +
 .../properties/IPartitioningProperty.java       |   81 ++
 .../IPartitioningRequirementsCoordinator.java   |   98 ++
 .../properties/IPhysicalPropertiesVector.java   |   44 +
 .../algebra/properties/IPropertiesComputer.java |   23 +
 .../algebra/properties/IStructuralProperty.java |   23 +
 .../properties/LocalGroupingProperty.java       |  103 ++
 .../algebra/properties/LocalOrderProperty.java  |  210 +++
 .../properties/LogicalPropertiesVectorImpl.java |   43 +
 .../core/algebra/properties/OrderColumn.java    |   60 +
 .../properties/OrderedPartitionedProperty.java  |   80 ++
 .../properties/PhysicalRequirements.java        |   34 +
 .../core/algebra/properties/PropertiesUtil.java |  320 +++++
 .../properties/RandomPartitioningProperty.java  |   61 +
 .../algebra/properties/ResultSetDomain.java     |   27 +
 .../properties/StructuralPropertiesVector.java  |  101 ++
 .../properties/TypePropagationPolicy.java       |   97 ++
 .../UnorderedPartitionedProperty.java           |   64 +
 .../UnpartitionedPropertyComputer.java          |   31 +
 .../properties/VariablePropagationPolicy.java   |   78 ++
 .../algebra/scripting/IScriptDescription.java   |   30 +
 .../StringStreamingScriptDescription.java       |   66 +
 .../algebra/typing/AbstractTypeEnvironment.java |   59 +
 .../core/algebra/typing/ITypeEnvPointer.java    |   21 +
 .../core/algebra/typing/ITypingContext.java     |   33 +
 .../typing/NonPropagatingTypeEnvironment.java   |   42 +
 .../algebra/typing/OpRefTypeEnvPointer.java     |   42 +
 .../PropagateOperatorInputsTypeEnvironment.java |   76 +
 .../typing/PropagatingTypeEnvironment.java      |   87 ++
 .../algebra/util/OperatorManipulationUtil.java  |  225 +++
 .../algebra/util/OperatorPropertiesUtil.java    |  300 ++++
 .../visitors/AbstractConstVarFunVisitor.java    |   46 +
 .../ILogicalExpressionReferenceTransform.java   |   25 +
 .../visitors/ILogicalExpressionVisitor.java     |   39 +
 .../visitors/ILogicalOperatorVisitor.java       |  117 ++
 .../core/config/AlgebricksConfig.java           |   23 +
 .../algebricks/core/config/SysoutFormatter.java |   35 +
 .../impl/ConnectorPolicyAssignmentPolicy.java   |   38 +
 .../algebricks/core/jobgen/impl/JobBuilder.java |  330 +++++
 .../core/jobgen/impl/JobGenContext.java         |  208 +++
 .../core/jobgen/impl/JobGenHelper.java          |  182 +++
 .../core/jobgen/impl/OperatorSchemaImpl.java    |   96 ++
 .../core/jobgen/impl/PlanCompiler.java          |  125 ++
 .../rewriter/base/AbstractRuleController.java   |  130 ++
 .../base/AlgebricksOptimizationContext.java     |  299 ++++
 .../core/rewriter/base/HeuristicOptimizer.java  |  125 ++
 .../rewriter/base/IAlgebraicRewriteRule.java    |   28 +
 .../base/IOptimizationContextFactory.java       |   29 +
 .../base/PhysicalOptimizationConfig.java        |  156 +++
 .../algebricks/core/utils/Substitution.java     |   25 +
 .../ics/hyracks/algebricks/data/IAWriter.java   |   25 -
 .../algebricks/data/IAWriterFactory.java        |   25 -
 .../data/IBinaryBooleanInspector.java           |   19 -
 .../data/IBinaryBooleanInspectorFactory.java    |   23 -
 .../data/IBinaryComparatorFactoryProvider.java  |   23 -
 .../IBinaryHashFunctionFactoryProvider.java     |   22 -
 .../data/IBinaryHashFunctionFamilyProvider.java |   25 -
 .../data/IBinaryIntegerInspector.java           |   21 -
 .../data/IBinaryIntegerInspectorFactory.java    |   23 -
 .../ILinearizeComparatorFactoryProvider.java    |   23 -
 .../INormalizedKeyComputerFactoryProvider.java  |   21 -
 .../ics/hyracks/algebricks/data/IPrinter.java   |   25 -
 .../algebricks/data/IPrinterFactory.java        |   21 -
 .../data/IPrinterFactoryProvider.java           |   21 -
 .../data/IResultSerializerFactoryProvider.java  |   37 -
 .../data/ISerializerDeserializerProvider.java   |   23 -
 .../algebricks/data/ITypeTraitProvider.java     |   21 -
 .../data/impl/BinaryBooleanInspectorImpl.java   |   38 -
 .../data/impl/BinaryIntegerInspectorImpl.java   |   39 -
 .../data/impl/IntegerPrinterFactory.java        |   54 -
 .../data/impl/NoopNullWriterFactory.java        |   43 -
 .../data/impl/UTF8StringPrinterFactory.java     |   63 -
 .../algebricks/data/utils/WriteValueTools.java  |  120 --
 .../hyracks/algebricks/data/IAWriter.java       |   25 +
 .../algebricks/data/IAWriterFactory.java        |   25 +
 .../data/IBinaryBooleanInspector.java           |   19 +
 .../data/IBinaryBooleanInspectorFactory.java    |   23 +
 .../data/IBinaryComparatorFactoryProvider.java  |   23 +
 .../IBinaryHashFunctionFactoryProvider.java     |   22 +
 .../data/IBinaryHashFunctionFamilyProvider.java |   25 +
 .../data/IBinaryIntegerInspector.java           |   21 +
 .../data/IBinaryIntegerInspectorFactory.java    |   23 +
 .../ILinearizeComparatorFactoryProvider.java    |   23 +
 .../INormalizedKeyComputerFactoryProvider.java  |   21 +
 .../hyracks/algebricks/data/IPrinter.java       |   25 +
 .../algebricks/data/IPrinterFactory.java        |   21 +
 .../data/IPrinterFactoryProvider.java           |   21 +
 .../data/IResultSerializerFactoryProvider.java  |   37 +
 .../data/ISerializerDeserializerProvider.java   |   23 +
 .../algebricks/data/ITypeTraitProvider.java     |   21 +
 .../data/impl/BinaryBooleanInspectorImpl.java   |   38 +
 .../data/impl/BinaryIntegerInspectorImpl.java   |   39 +
 .../data/impl/IntegerPrinterFactory.java        |   54 +
 .../data/impl/NoopNullWriterFactory.java        |   43 +
 .../data/impl/UTF8StringPrinterFactory.java     |   63 +
 .../algebricks/data/utils/WriteValueTools.java  |  120 ++
 .../algebricks/examples/piglet/ast/ASTNode.java |   30 -
 .../examples/piglet/ast/AssignmentNode.java     |   39 -
 .../examples/piglet/ast/DumpNode.java           |   38 -
 .../examples/piglet/ast/ExpressionNode.java     |   18 -
 .../piglet/ast/FieldAccessExpressionNode.java   |   39 -
 .../examples/piglet/ast/FilterNode.java         |   39 -
 .../examples/piglet/ast/FunctionTag.java        |   34 -
 .../piglet/ast/LiteralExpressionNode.java       |   41 -
 .../examples/piglet/ast/LoadNode.java           |   41 -
 .../examples/piglet/ast/RelationNode.java       |   18 -
 .../ast/ScalarFunctionExpressionNode.java       |   48 -
 .../examples/piglet/compiler/ConstantValue.java |   52 -
 .../piglet/compiler/PigletCompiler.java         |  376 -----
 .../compiler/PigletPrinterFactoryProvider.java  |  105 --
 .../piglet/exceptions/PigletException.java      |   23 -
 .../piglet/metadata/FileSplitUtils.java         |   36 -
 .../piglet/metadata/PigletFileDataSink.java     |   54 -
 .../piglet/metadata/PigletFileDataSource.java   |   76 -
 .../piglet/metadata/PigletFunction.java         |   34 -
 .../piglet/metadata/PigletMetadataProvider.java |  234 ----
 .../piglet/rewriter/PigletRewriteRuleset.java   |  131 --
 .../piglet/runtime/PigletExpressionJobGen.java  |  140 --
 .../IPigletFunctionEvaluatorFactoryBuilder.java |   22 -
 .../IntegerEqFunctionEvaluatorFactory.java      |   67 -
 .../functions/PigletFunctionRegistry.java       |   50 -
 .../examples/piglet/types/BagType.java          |   22 -
 .../examples/piglet/types/CharArrayType.java    |   27 -
 .../examples/piglet/types/DoubleType.java       |   27 -
 .../examples/piglet/types/FloatType.java        |   27 -
 .../examples/piglet/types/IntegerType.java      |   27 -
 .../examples/piglet/types/LongType.java         |   27 -
 .../examples/piglet/types/MapType.java          |   22 -
 .../examples/piglet/types/Schema.java           |   31 -
 .../examples/piglet/types/TupleType.java        |   22 -
 .../algebricks/examples/piglet/types/Type.java  |   30 -
 .../algebricks/examples/piglet/ast/ASTNode.java |   30 +
 .../examples/piglet/ast/AssignmentNode.java     |   39 +
 .../examples/piglet/ast/DumpNode.java           |   38 +
 .../examples/piglet/ast/ExpressionNode.java     |   18 +
 .../piglet/ast/FieldAccessExpressionNode.java   |   39 +
 .../examples/piglet/ast/FilterNode.java         |   39 +
 .../examples/piglet/ast/FunctionTag.java        |   34 +
 .../piglet/ast/LiteralExpressionNode.java       |   41 +
 .../examples/piglet/ast/LoadNode.java           |   41 +
 .../examples/piglet/ast/RelationNode.java       |   18 +
 .../ast/ScalarFunctionExpressionNode.java       |   48 +
 .../examples/piglet/compiler/ConstantValue.java |   52 +
 .../piglet/compiler/PigletCompiler.java         |  376 +++++
 .../compiler/PigletPrinterFactoryProvider.java  |  105 ++
 .../piglet/exceptions/PigletException.java      |   23 +
 .../piglet/metadata/FileSplitUtils.java         |   36 +
 .../piglet/metadata/PigletFileDataSink.java     |   54 +
 .../piglet/metadata/PigletFileDataSource.java   |   76 +
 .../piglet/metadata/PigletFunction.java         |   34 +
 .../piglet/metadata/PigletMetadataProvider.java |  234 ++++
 .../piglet/rewriter/PigletRewriteRuleset.java   |  131 ++
 .../piglet/runtime/PigletExpressionJobGen.java  |  140 ++
 .../IPigletFunctionEvaluatorFactoryBuilder.java |   22 +
 .../IntegerEqFunctionEvaluatorFactory.java      |   67 +
 .../functions/PigletFunctionRegistry.java       |   50 +
 .../examples/piglet/types/BagType.java          |   22 +
 .../examples/piglet/types/CharArrayType.java    |   27 +
 .../examples/piglet/types/DoubleType.java       |   27 +
 .../examples/piglet/types/FloatType.java        |   27 +
 .../examples/piglet/types/IntegerType.java      |   27 +
 .../examples/piglet/types/LongType.java         |   27 +
 .../examples/piglet/types/MapType.java          |   22 +
 .../examples/piglet/types/Schema.java           |   31 +
 .../examples/piglet/types/TupleType.java        |   22 +
 .../algebricks/examples/piglet/types/Type.java  |   30 +
 .../examples/piglet/test/PigletTest.java        |   43 -
 .../examples/piglet/test/PigletTestCase.java    |   56 -
 .../examples/piglet/test/PigletTest.java        |   43 +
 .../examples/piglet/test/PigletTestCase.java    |   56 +
 .../rules/AbstractDecorrelationRule.java        |  104 --
 .../rewriter/rules/AbstractExtractExprRule.java |   44 -
 .../rules/AbstractIntroduceCombinerRule.java    |  158 ---
 .../AbstractIntroduceGroupByCombinerRule.java   |  340 -----
 .../rules/BreakSelectIntoConjunctsRule.java     |   82 --
 .../rules/ComplexJoinInferenceRule.java         |  138 --
 .../rules/ComplexUnnestToProductRule.java       |  314 -----
 .../rewriter/rules/ConsolidateAssignsRule.java  |   73 -
 .../rewriter/rules/ConsolidateSelectsRule.java  |  109 --
 .../rewriter/rules/CopyLimitDownRule.java       |  105 --
 .../rules/EliminateGroupByEmptyKeyRule.java     |   86 --
 .../rewriter/rules/EliminateSubplanRule.java    |  127 --
 ...inateSubplanWithInputCardinalityOneRule.java |  199 ---
 .../rules/EnforceOrderByAfterSubplan.java       |  165 ---
 .../rules/EnforceStructuralPropertiesRule.java  |  614 --------
 .../rules/ExtractCommonExpressionsRule.java     |  436 ------
 .../rules/ExtractCommonOperatorsRule.java       |  502 -------
 .../ExtractFunctionsFromJoinConditionRule.java  |  155 ---
 .../rules/ExtractGbyExpressionsRule.java        |  111 --
 .../FactorRedundantGroupAndDecorVarsRule.java   |   92 --
 .../rewriter/rules/InferTypesRule.java          |   42 -
 .../rules/InlineAssignIntoAggregateRule.java    |  136 --
 .../InlineSingleReferenceVariablesRule.java     |   94 --
 .../rewriter/rules/InlineVariablesRule.java     |  253 ----
 .../rewriter/rules/InsertOuterJoinRule.java     |  121 --
 .../rules/InsertProjectBeforeUnionRule.java     |  110 --
 .../rules/IntroHashPartitionMergeExchange.java  |   74 -
 .../rules/IntroJoinInsideSubplanRule.java       |  102 --
 .../rules/IntroduceAggregateCombinerRule.java   |   57 -
 .../rules/IntroduceGroupByCombinerRule.java     |   31 -
 .../rules/IntroduceGroupByForSubplanRule.java   |  323 -----
 .../rewriter/rules/IntroduceProjectsRule.java   |  171 ---
 .../rules/IsolateHyracksOperatorsRule.java      |  140 --
 ...oveFreeVariableOperatorOutOfSubplanRule.java |  183 ---
 .../rewriter/rules/NestedSubplanToJoinRule.java |  136 --
 .../rewriter/rules/PullSelectOutOfEqJoin.java   |  114 --
 .../rules/PushAssignBelowUnionAllRule.java      |  165 ---
 .../rules/PushAssignDownThroughProductRule.java |   87 --
 .../rewriter/rules/PushFunctionsBelowJoin.java  |  208 ---
 .../rewriter/rules/PushGroupByIntoSortRule.java |  150 --
 .../PushMapOperatorDownThroughProductRule.java  |   87 --
 ...hNestedOrderByUnderPreSortedGroupByRule.java |  119 --
 .../rewriter/rules/PushProjectDownRule.java     |  219 ---
 .../PushProjectIntoDataSourceScanRule.java      |   60 -
 .../rewriter/rules/PushSelectDownRule.java      |   98 --
 .../rewriter/rules/PushSelectIntoJoinRule.java  |  309 -----
 .../rules/PushSubplanIntoGroupByRule.java       |  217 ---
 ...planWithAggregateDownThroughProductRule.java |  104 --
 .../rules/PushUnnestDownThroughProductRule.java |   87 --
 .../rules/PushUnnestDownThroughUnionRule.java   |  125 --
 .../rewriter/rules/ReinferAllTypesRule.java     |   64 -
 .../rules/RemoveRedundantGroupByDecorVars.java  |   81 --
 .../rules/RemoveRedundantProjectionRule.java    |   89 --
 .../rules/RemoveRedundantVariablesRule.java     |  299 ----
 .../RemoveUnusedAssignAndAggregateRule.java     |  240 ----
 .../SetAlgebricksPhysicalOperatorsRule.java     |  424 ------
 .../rewriter/rules/SetExecutionModeRule.java    |   70 -
 .../rules/SimpleUnnestToProductRule.java        |  144 --
 .../rewriter/rules/SubplanOutOfGroupRule.java   |  121 --
 .../algebricks/rewriter/util/JoinUtils.java     |  210 ---
 .../util/PhysicalOptimizationsUtil.java         |   71 -
 .../rules/AbstractDecorrelationRule.java        |  104 ++
 .../rewriter/rules/AbstractExtractExprRule.java |   44 +
 .../rules/AbstractIntroduceCombinerRule.java    |  158 +++
 .../AbstractIntroduceGroupByCombinerRule.java   |  340 +++++
 .../rules/BreakSelectIntoConjunctsRule.java     |   82 ++
 .../rules/ComplexJoinInferenceRule.java         |  138 ++
 .../rules/ComplexUnnestToProductRule.java       |  314 +++++
 .../rewriter/rules/ConsolidateAssignsRule.java  |   73 +
 .../rewriter/rules/ConsolidateSelectsRule.java  |  109 ++
 .../rewriter/rules/CopyLimitDownRule.java       |  105 ++
 .../rules/EliminateGroupByEmptyKeyRule.java     |   86 ++
 .../rewriter/rules/EliminateSubplanRule.java    |  127 ++
 ...inateSubplanWithInputCardinalityOneRule.java |  199 +++
 .../rules/EnforceOrderByAfterSubplan.java       |  165 +++
 .../rules/EnforceStructuralPropertiesRule.java  |  614 ++++++++
 .../rules/ExtractCommonExpressionsRule.java     |  436 ++++++
 .../rules/ExtractCommonOperatorsRule.java       |  502 +++++++
 .../ExtractFunctionsFromJoinConditionRule.java  |  155 +++
 .../rules/ExtractGbyExpressionsRule.java        |  111 ++
 .../FactorRedundantGroupAndDecorVarsRule.java   |   92 ++
 .../rewriter/rules/InferTypesRule.java          |   42 +
 .../rules/InlineAssignIntoAggregateRule.java    |  136 ++
 .../InlineSingleReferenceVariablesRule.java     |   94 ++
 .../rewriter/rules/InlineVariablesRule.java     |  253 ++++
 .../rewriter/rules/InsertOuterJoinRule.java     |  121 ++
 .../rules/InsertProjectBeforeUnionRule.java     |  110 ++
 .../rules/IntroHashPartitionMergeExchange.java  |   74 +
 .../rules/IntroJoinInsideSubplanRule.java       |  102 ++
 .../rules/IntroduceAggregateCombinerRule.java   |   57 +
 .../rules/IntroduceGroupByCombinerRule.java     |   31 +
 .../rules/IntroduceGroupByForSubplanRule.java   |  323 +++++
 .../rewriter/rules/IntroduceProjectsRule.java   |  171 +++
 .../rules/IsolateHyracksOperatorsRule.java      |  140 ++
 ...oveFreeVariableOperatorOutOfSubplanRule.java |  183 +++
 .../rewriter/rules/NestedSubplanToJoinRule.java |  136 ++
 .../rewriter/rules/PullSelectOutOfEqJoin.java   |  114 ++
 .../rules/PushAssignBelowUnionAllRule.java      |  165 +++
 .../rules/PushAssignDownThroughProductRule.java |   87 ++
 .../rewriter/rules/PushFunctionsBelowJoin.java  |  208 +++
 .../rewriter/rules/PushGroupByIntoSortRule.java |  150 ++
 .../PushMapOperatorDownThroughProductRule.java  |   87 ++
 ...hNestedOrderByUnderPreSortedGroupByRule.java |  119 ++
 .../rewriter/rules/PushProjectDownRule.java     |  219 +++
 .../PushProjectIntoDataSourceScanRule.java      |   60 +
 .../rewriter/rules/PushSelectDownRule.java      |   98 ++
 .../rewriter/rules/PushSelectIntoJoinRule.java  |  309 +++++
 .../rules/PushSubplanIntoGroupByRule.java       |  217 +++
 ...planWithAggregateDownThroughProductRule.java |  104 ++
 .../rules/PushUnnestDownThroughProductRule.java |   87 ++
 .../rules/PushUnnestDownThroughUnionRule.java   |  125 ++
 .../rewriter/rules/ReinferAllTypesRule.java     |   64 +
 .../rules/RemoveRedundantGroupByDecorVars.java  |   81 ++
 .../rules/RemoveRedundantProjectionRule.java    |   89 ++
 .../rules/RemoveRedundantVariablesRule.java     |  299 ++++
 .../RemoveUnusedAssignAndAggregateRule.java     |  240 ++++
 .../SetAlgebricksPhysicalOperatorsRule.java     |  424 ++++++
 .../rewriter/rules/SetExecutionModeRule.java    |   70 +
 .../rules/SimpleUnnestToProductRule.java        |  144 ++
 .../rewriter/rules/SubplanOutOfGroupRule.java   |  121 ++
 .../algebricks/rewriter/util/JoinUtils.java     |  210 +++
 .../util/PhysicalOptimizationsUtil.java         |   71 +
 .../TupleCountAggregateFunctionFactory.java     |   66 -
 ...pleCountRunningAggregateFunctionFactory.java |   56 -
 .../runtime/base/AlgebricksPipeline.java        |   49 -
 .../runtime/base/IAggregateEvaluator.java       |   29 -
 .../base/IAggregateEvaluatorFactory.java        |   24 -
 .../runtime/base/ICopyAggregateFunction.java    |   29 -
 .../base/ICopyAggregateFunctionFactory.java     |   24 -
 .../algebricks/runtime/base/ICopyEvaluator.java |   22 -
 .../runtime/base/ICopyEvaluatorFactory.java     |   24 -
 .../base/ICopyRunningAggregateFunction.java     |   24 -
 .../ICopyRunningAggregateFunctionFactory.java   |   25 -
 .../ICopySerializableAggregateFunction.java     |   58 -
 ...opySerializableAggregateFunctionFactory.java |   23 -
 .../runtime/base/ICopyUnnestingFunction.java    |   25 -
 .../base/ICopyUnnestingFunctionFactory.java     |   24 -
 .../algebricks/runtime/base/IPushRuntime.java   |   24 -
 .../runtime/base/IPushRuntimeFactory.java       |   25 -
 .../base/IRunningAggregateEvaluator.java        |   25 -
 .../base/IRunningAggregateEvaluatorFactory.java |   23 -
 .../runtime/base/IScalarEvaluator.java          |   23 -
 .../runtime/base/IScalarEvaluatorFactory.java   |   24 -
 .../runtime/base/IUnnestingEvaluator.java       |   25 -
 .../base/IUnnestingEvaluatorFactory.java        |   24 -
 .../runtime/base/IUnnestingPositionWriter.java  |   22 -
 .../evaluators/ColumnAccessEvalFactory.java     |   61 -
 .../runtime/evaluators/ConstantEvalFactory.java |   57 -
 .../evaluators/ConstantEvaluatorFactory.java    |   48 -
 .../evaluators/TupleFieldEvaluatorFactory.java  |   43 -
 .../aggreg/AggregateRuntimeFactory.java         |  135 --
 ...estedPlansAccumulatingAggregatorFactory.java |  232 ----
 .../NestedPlansRunningAggregatorFactory.java    |  242 ----
 ...SerializableAggregatorDescriptorFactory.java |  163 ---
 ...AlgebricksAccumulatingAggregatorFactory.java |  143 --
 ...eInputOneOutputOneFieldFramePushRuntime.java |   55 -
 ...actOneInputOneOutputOneFramePushRuntime.java |  102 --
 .../AbstractOneInputOneOutputPushRuntime.java   |   28 -
 ...AbstractOneInputOneOutputRuntimeFactory.java |   41 -
 .../base/AbstractOneInputPushRuntime.java       |   38 -
 .../base/AbstractOneInputSinkPushRuntime.java   |   34 -
 .../base/AbstractOneInputSourcePushRuntime.java |   42 -
 .../operators/base/SinkRuntimeFactory.java      |   59 -
 .../MicroPreClusteredGroupRuntimeFactory.java   |   92 --
 .../meta/AlgebricksMetaOperatorDescriptor.java  |  150 --
 .../operators/meta/PipelineAssembler.java       |   66 -
 .../operators/meta/SubplanRuntimeFactory.java   |  165 ---
 .../sort/InMemorySortRuntimeFactory.java        |   92 --
 .../operators/std/AssignRuntimeFactory.java     |  161 ---
 .../std/EmptyTupleSourceRuntimeFactory.java     |   57 -
 .../std/NestedTupleSourceRuntimeFactory.java    |   72 -
 .../PartitioningSplitOperatorDescriptor.java    |  168 ---
 .../operators/std/PrinterRuntimeFactory.java    |   60 -
 .../std/RunningAggregateRuntimeFactory.java     |  145 --
 .../operators/std/SinkWriterRuntime.java        |   94 --
 .../operators/std/SinkWriterRuntimeFactory.java |   76 -
 .../std/StreamLimitRuntimeFactory.java          |  154 ---
 .../std/StreamProjectRuntimeFactory.java        |   85 --
 .../std/StreamSelectRuntimeFactory.java         |  138 --
 .../std/StringStreamingRuntimeFactory.java      |  189 ---
 .../operators/std/UnnestRuntimeFactory.java     |  166 ---
 .../ResultSerializerFactoryProvider.java        |   77 --
 .../writers/PrinterBasedWriterFactory.java      |   68 -
 .../writers/SerializedDataWriterFactory.java    |   63 -
 .../TupleCountAggregateFunctionFactory.java     |   66 +
 ...pleCountRunningAggregateFunctionFactory.java |   56 +
 .../runtime/base/AlgebricksPipeline.java        |   49 +
 .../runtime/base/IAggregateEvaluator.java       |   29 +
 .../base/IAggregateEvaluatorFactory.java        |   24 +
 .../runtime/base/ICopyAggregateFunction.java    |   29 +
 .../base/ICopyAggregateFunctionFactory.java     |   24 +
 .../algebricks/runtime/base/ICopyEvaluator.java |   22 +
 .../runtime/base/ICopyEvaluatorFactory.java     |   24 +
 .../base/ICopyRunningAggregateFunction.java     |   24 +
 .../ICopyRunningAggregateFunctionFactory.java   |   25 +
 .../ICopySerializableAggregateFunction.java     |   58 +
 ...opySerializableAggregateFunctionFactory.java |   23 +
 .../runtime/base/ICopyUnnestingFunction.java    |   25 +
 .../base/ICopyUnnestingFunctionFactory.java     |   24 +
 .../algebricks/runtime/base/IPushRuntime.java   |   24 +
 .../runtime/base/IPushRuntimeFactory.java       |   25 +
 .../base/IRunningAggregateEvaluator.java        |   25 +
 .../base/IRunningAggregateEvaluatorFactory.java |   23 +
 .../runtime/base/IScalarEvaluator.java          |   23 +
 .../runtime/base/IScalarEvaluatorFactory.java   |   24 +
 .../runtime/base/IUnnestingEvaluator.java       |   25 +
 .../base/IUnnestingEvaluatorFactory.java        |   24 +
 .../runtime/base/IUnnestingPositionWriter.java  |   22 +
 .../evaluators/ColumnAccessEvalFactory.java     |   61 +
 .../runtime/evaluators/ConstantEvalFactory.java |   57 +
 .../evaluators/ConstantEvaluatorFactory.java    |   48 +
 .../evaluators/TupleFieldEvaluatorFactory.java  |   43 +
 .../aggreg/AggregateRuntimeFactory.java         |  135 ++
 ...estedPlansAccumulatingAggregatorFactory.java |  232 ++++
 .../NestedPlansRunningAggregatorFactory.java    |  242 ++++
 ...SerializableAggregatorDescriptorFactory.java |  163 +++
 ...AlgebricksAccumulatingAggregatorFactory.java |  143 ++
 ...eInputOneOutputOneFieldFramePushRuntime.java |   55 +
 ...actOneInputOneOutputOneFramePushRuntime.java |  102 ++
 .../AbstractOneInputOneOutputPushRuntime.java   |   28 +
 ...AbstractOneInputOneOutputRuntimeFactory.java |   41 +
 .../base/AbstractOneInputPushRuntime.java       |   38 +
 .../base/AbstractOneInputSinkPushRuntime.java   |   34 +
 .../base/AbstractOneInputSourcePushRuntime.java |   42 +
 .../operators/base/SinkRuntimeFactory.java      |   59 +
 .../MicroPreClusteredGroupRuntimeFactory.java   |   92 ++
 .../meta/AlgebricksMetaOperatorDescriptor.java  |  150 ++
 .../operators/meta/PipelineAssembler.java       |   66 +
 .../operators/meta/SubplanRuntimeFactory.java   |  165 +++
 .../sort/InMemorySortRuntimeFactory.java        |   92 ++
 .../operators/std/AssignRuntimeFactory.java     |  161 +++
 .../std/EmptyTupleSourceRuntimeFactory.java     |   57 +
 .../std/NestedTupleSourceRuntimeFactory.java    |   72 +
 .../PartitioningSplitOperatorDescriptor.java    |  168 +++
 .../operators/std/PrinterRuntimeFactory.java    |   60 +
 .../std/RunningAggregateRuntimeFactory.java     |  145 ++
 .../operators/std/SinkWriterRuntime.java        |   94 ++
 .../operators/std/SinkWriterRuntimeFactory.java |   76 +
 .../std/StreamLimitRuntimeFactory.java          |  154 +++
 .../std/StreamProjectRuntimeFactory.java        |   85 ++
 .../std/StreamSelectRuntimeFactory.java         |  138 ++
 .../std/StringStreamingRuntimeFactory.java      |  189 +++
 .../operators/std/UnnestRuntimeFactory.java     |  166 +++
 .../ResultSerializerFactoryProvider.java        |   77 ++
 .../writers/PrinterBasedWriterFactory.java      |   68 +
 .../writers/SerializedDataWriterFactory.java    |   63 +
 .../tests/pushruntime/IntArrayUnnester.java     |   73 -
 .../pushruntime/IntegerAddEvalFactory.java      |   69 -
 .../pushruntime/IntegerConstantEvalFactory.java |   62 -
 .../pushruntime/IntegerEqualsEvalFactory.java   |   58 -
 .../IntegerGreaterThanEvalFactory.java          |   58 -
 .../tests/script/IdentityStreamingScript.java   |   32 -
 .../tests/pushruntime/IntArrayUnnester.java     |   73 +
 .../pushruntime/IntegerAddEvalFactory.java      |   69 +
 .../pushruntime/IntegerConstantEvalFactory.java |   62 +
 .../pushruntime/IntegerEqualsEvalFactory.java   |   58 +
 .../IntegerGreaterThanEvalFactory.java          |   58 +
 .../tests/script/IdentityStreamingScript.java   |   32 +
 .../tests/pushruntime/PushRuntimeTest.java      |  913 ------------
 .../algebricks/tests/tools/WriteValueTest.java  |  111 --
 .../util/AlgebricksHyracksIntegrationUtil.java  |   88 --
 .../tests/pushruntime/PushRuntimeTest.java      |  913 ++++++++++++
 .../algebricks/tests/tools/WriteValueTest.java  |  111 ++
 .../util/AlgebricksHyracksIntegrationUtil.java  |   88 ++
 .../api/application/IApplicationContext.java    |   47 -
 .../api/application/ICCApplicationContext.java  |   62 -
 .../application/ICCApplicationEntryPoint.java   |   21 -
 .../application/IClusterLifecycleListener.java  |   39 -
 .../api/application/INCApplicationContext.java  |   76 -
 .../application/INCApplicationEntryPoint.java   |   23 -
 .../api/application/IStateDumpHandler.java      |   22 -
 .../ics/hyracks/api/channels/IInputChannel.java |   36 -
 .../api/channels/IInputChannelMonitor.java      |   23 -
 .../api/client/ClusterControllerInfo.java       |   45 -
 .../client/HyracksClientInterfaceFunctions.java |  293 ----
 .../HyracksClientInterfaceRemoteProxy.java      |  130 --
 .../hyracks/api/client/HyracksConnection.java   |  189 ---
 .../api/client/IHyracksClientConnection.java    |  187 ---
 .../api/client/IHyracksClientInterface.java     |   55 -
 .../hyracks/api/client/NodeControllerInfo.java  |   55 -
 .../uci/ics/hyracks/api/client/NodeStatus.java  |   20 -
 .../impl/ActivityClusterGraphBuilder.java       |  176 ---
 .../impl/IConnectorDescriptorVisitor.java       |   22 -
 .../client/impl/IOperatorDescriptorVisitor.java |   22 -
 .../client/impl/JobActivityGraphBuilder.java    |  141 --
 ...ionActivityClusterGraphGeneratorFactory.java |  113 --
 .../ics/hyracks/api/client/impl/PlanUtils.java  |   47 -
 .../ics/hyracks/api/comm/FixedSizeFrame.java    |   59 -
 .../ics/hyracks/api/comm/FrameConstants.java    |   48 -
 .../uci/ics/hyracks/api/comm/FrameHelper.java   |   78 --
 .../edu/uci/ics/hyracks/api/comm/IFrame.java    |   60 -
 .../ics/hyracks/api/comm/IFrameAppender.java    |   55 -
 .../hyracks/api/comm/IFrameFieldAppender.java   |   47 -
 .../uci/ics/hyracks/api/comm/IFrameReader.java  |   25 -
 .../hyracks/api/comm/IFrameTupleAccessor.java   |   44 -
 .../hyracks/api/comm/IFrameTupleAppender.java   |   39 -
 .../uci/ics/hyracks/api/comm/IFrameWriter.java  |   79 --
 .../hyracks/api/comm/IPartitionCollector.java   |   42 -
 .../api/comm/IPartitionWriterFactory.java       |   21 -
 .../ics/hyracks/api/comm/NetworkAddress.java    |   98 --
 .../hyracks/api/comm/NoShrinkVSizeFrame.java    |   31 -
 .../ics/hyracks/api/comm/PartitionChannel.java  |   37 -
 .../uci/ics/hyracks/api/comm/VSizeFrame.java    |   76 -
 .../ics/hyracks/api/constraints/Constraint.java |   46 -
 .../api/constraints/IConstraintAcceptor.java    |   19 -
 .../constraints/PartitionConstraintHelper.java  |   44 -
 .../expressions/ConstantExpression.java         |   45 -
 .../expressions/ConstraintExpression.java       |   41 -
 .../expressions/LValueConstraintExpression.java |   19 -
 .../expressions/PartitionCountExpression.java   |   72 -
 .../PartitionLocationExpression.java            |   81 --
 .../uci/ics/hyracks/api/context/ICCContext.java |   30 -
 .../api/context/IHyracksCommonContext.java      |   22 -
 .../api/context/IHyracksFrameMgrContext.java    |   39 -
 .../api/context/IHyracksJobletContext.java      |   35 -
 .../api/context/IHyracksRootContext.java        |   26 -
 .../api/context/IHyracksTaskContext.java        |   36 -
 .../ics/hyracks/api/dataflow/ActivityId.java    |   94 --
 .../api/dataflow/ConnectorDescriptorId.java     |   78 --
 .../uci/ics/hyracks/api/dataflow/IActivity.java |   28 -
 .../api/dataflow/IActivityGraphBuilder.java     |   25 -
 .../api/dataflow/IConnectorDescriptor.java      |  135 --
 .../ics/hyracks/api/dataflow/IDataReader.java   |   39 -
 .../ics/hyracks/api/dataflow/IDataWriter.java   |   48 -
 .../api/dataflow/IOpenableDataReader.java       |   21 -
 .../api/dataflow/IOpenableDataWriter.java       |   21 -
 .../api/dataflow/IOperatorDescriptor.java       |   93 --
 .../api/dataflow/IOperatorNodePullable.java     |   19 -
 .../api/dataflow/IOperatorNodePushable.java     |   34 -
 .../api/dataflow/OperatorDescriptorId.java      |   85 --
 .../api/dataflow/OperatorInstanceId.java        |   65 -
 .../ics/hyracks/api/dataflow/TaskAttemptId.java |   93 --
 .../uci/ics/hyracks/api/dataflow/TaskId.java    |   93 --
 .../connectors/ConnectorPolicyFactory.java      |   66 -
 .../dataflow/connectors/IConnectorPolicy.java   |   27 -
 .../IConnectorPolicyAssignmentPolicy.java       |   24 -
 .../connectors/PipeliningConnectorPolicy.java   |   39 -
 ...SideMaterializedBlockingConnectorPolicy.java |   39 -
 ...deMaterializedPipeliningConnectorPolicy.java |   39 -
 ...SideMaterializedBlockingConnectorPolicy.java |   39 -
 ...deMaterializedPipeliningConnectorPolicy.java |   39 -
 ...SideMaterializedBlockingConnectorPolicy.java |   40 -
 .../api/dataflow/state/IStateObject.java        |   33 -
 .../value/BinaryComparatorConstant.java         |   30 -
 .../api/dataflow/value/IBinaryComparator.java   |   21 -
 .../value/IBinaryComparatorFactory.java         |   21 -
 .../api/dataflow/value/IBinaryHashFunction.java |   21 -
 .../value/IBinaryHashFunctionFactory.java       |   21 -
 .../value/IBinaryHashFunctionFamily.java        |   21 -
 .../hyracks/api/dataflow/value/IComparator.java |   19 -
 .../api/dataflow/value/IComparatorFactory.java  |   21 -
 .../api/dataflow/value/IHashFunction.java       |   21 -
 .../dataflow/value/IHashFunctionFactory.java    |   21 -
 .../dataflow/value/ILinearizeComparator.java    |   20 -
 .../value/ILinearizeComparatorFactory.java      |   19 -
 .../dataflow/value/INormalizedKeyComputer.java  |   19 -
 .../value/INormalizedKeyComputerFactory.java    |   21 -
 .../hyracks/api/dataflow/value/INullWriter.java |   23 -
 .../api/dataflow/value/INullWriterFactory.java  |   21 -
 .../api/dataflow/value/IPredicateEvaluator.java |   25 -
 .../value/IPredicateEvaluatorFactory.java       |   26 -
 .../IPredicateEvaluatorFactoryProvider.java     |   26 -
 .../value/IRecordDescriptorProvider.java        |   23 -
 .../api/dataflow/value/IResultSerializer.java   |   38 -
 .../value/IResultSerializerFactory.java         |   29 -
 .../dataflow/value/ISerializerDeserializer.java |   48 -
 .../dataflow/value/ITuplePairComparator.java    |   25 -
 .../value/ITuplePairComparatorFactory.java      |   23 -
 .../dataflow/value/ITuplePartitionComputer.java |   22 -
 .../value/ITuplePartitionComputerFactory.java   |   21 -
 .../value/ITuplePartitionComputerFamily.java    |   21 -
 .../hyracks/api/dataflow/value/ITypeTraits.java |   23 -
 .../api/dataflow/value/JSONSerializable.java    |   27 -
 .../api/dataflow/value/RecordDescriptor.java    |   49 -
 .../api/dataset/DatasetDirectoryRecord.java     |  100 --
 .../hyracks/api/dataset/DatasetJobRecord.java   |   68 -
 .../dataset/IDatasetInputChannelMonitor.java    |   27 -
 .../hyracks/api/dataset/IDatasetManager.java    |   28 -
 .../api/dataset/IDatasetPartitionManager.java   |   45 -
 .../api/dataset/IDatasetStateRecord.java        |   19 -
 .../hyracks/api/dataset/IHyracksDataset.java    |   22 -
 ...yracksDatasetDirectoryServiceConnection.java |   47 -
 ...HyracksDatasetDirectoryServiceInterface.java |   47 -
 .../api/dataset/IHyracksDatasetReader.java      |   25 -
 .../ics/hyracks/api/dataset/ResultSetId.java    |   52 -
 .../hyracks/api/dataset/ResultSetMetaData.java  |   34 -
 .../hyracks/api/deployment/DeploymentId.java    |   78 --
 .../api/exceptions/HyracksDataException.java    |   44 -
 .../api/exceptions/HyracksException.java        |   36 -
 .../uci/ics/hyracks/api/io/FileReference.java   |   65 -
 .../edu/uci/ics/hyracks/api/io/IFileHandle.java |   22 -
 .../edu/uci/ics/hyracks/api/io/IIOFuture.java   |   23 -
 .../edu/uci/ics/hyracks/api/io/IIOManager.java  |   53 -
 .../uci/ics/hyracks/api/io/IODeviceHandle.java  |   47 -
 .../hyracks/api/io/IWorkspaceFileFactory.java   |   23 -
 .../edu/uci/ics/hyracks/api/io/IWritable.java   |   30 -
 .../ics/hyracks/api/job/ActivityCluster.java    |  232 ----
 .../hyracks/api/job/ActivityClusterGraph.java   |  172 ---
 .../ics/hyracks/api/job/ActivityClusterId.java  |   77 --
 .../api/job/IActivityClusterGraphGenerator.java |   25 -
 .../IActivityClusterGraphGeneratorFactory.java  |   28 -
 .../api/job/IConnectorDescriptorRegistry.java   |   22 -
 .../hyracks/api/job/IGlobalJobDataFactory.java  |   23 -
 .../hyracks/api/job/IJobLifecycleListener.java  |   25 -
 .../api/job/IJobSerializerDeserializer.java     |   70 -
 .../IJobSerializerDeserializerContainer.java    |   45 -
 .../hyracks/api/job/IJobletEventListener.java   |   21 -
 .../api/job/IJobletEventListenerFactory.java    |   23 -
 .../api/job/IOperatorDescriptorRegistry.java    |   22 -
 .../hyracks/api/job/IOperatorEnvironment.java   |   23 -
 .../ics/hyracks/api/job/JobActivityGraph.java   |  105 --
 .../edu/uci/ics/hyracks/api/job/JobFlag.java    |   19 -
 .../java/edu/uci/ics/hyracks/api/job/JobId.java |   84 --
 .../edu/uci/ics/hyracks/api/job/JobInfo.java    |   50 -
 .../api/job/JobSerializerDeserializer.java      |   64 -
 .../job/JobSerializerDeserializerContainer.java |   52 -
 .../ics/hyracks/api/job/JobSpecification.java   |  409 ------
 .../edu/uci/ics/hyracks/api/job/JobStatus.java  |   22 -
 .../api/job/profiling/counters/ICounter.java    |   49 -
 .../job/profiling/counters/ICounterContext.java |   35 -
 .../api/lifecycle/ILifeCycleComponent.java      |   27 -
 .../lifecycle/ILifeCycleComponentManager.java   |   35 -
 .../lifecycle/LifeCycleComponentManager.java    |  169 ---
 .../uci/ics/hyracks/api/messages/IMessage.java  |   24 -
 .../hyracks/api/messages/IMessageBroker.java    |   24 -
 .../ics/hyracks/api/partitions/IPartition.java  |   27 -
 .../ics/hyracks/api/partitions/PartitionId.java |  127 --
 .../api/partitions/ResultSetPartitionId.java    |   87 --
 .../api/replication/IIOReplicationManager.java  |   27 -
 .../api/replication/IReplicationJob.java        |   45 -
 .../impl/AbstractReplicationJob.java            |   54 -
 .../hyracks/api/resources/IDeallocatable.java   |   19 -
 .../api/resources/IDeallocatableRegistry.java   |   19 -
 .../api/resources/memory/IMemoryManager.java    |   25 -
 .../rewriter/ActivityClusterGraphRewriter.java  |  381 -----
 .../OneToOneConnectedActivityCluster.java       |  122 --
 .../api/rewriter/runtime/SuperActivity.java     |  177 ---
 .../SuperActivityOperatorNodePushable.java      |  196 ---
 .../hyracks/api/topology/ClusterTopology.java   |   36 -
 .../hyracks/api/topology/NetworkEndpoint.java   |   46 -
 .../ics/hyracks/api/topology/NetworkSwitch.java |  108 --
 .../hyracks/api/topology/NetworkTerminal.java   |   30 -
 .../api/topology/TopologyDefinitionParser.java  |  135 --
 .../api/util/JavaSerializationUtils.java        |  117 --
 .../api/application/IApplicationContext.java    |   47 +
 .../api/application/ICCApplicationContext.java  |   62 +
 .../application/ICCApplicationEntryPoint.java   |   21 +
 .../application/IClusterLifecycleListener.java  |   39 +
 .../api/application/INCApplicationContext.java  |   76 +
 .../application/INCApplicationEntryPoint.java   |   23 +
 .../api/application/IStateDumpHandler.java      |   22 +
 .../hyracks/api/channels/IInputChannel.java     |   36 +
 .../api/channels/IInputChannelMonitor.java      |   23 +
 .../api/client/ClusterControllerInfo.java       |   45 +
 .../client/HyracksClientInterfaceFunctions.java |  293 ++++
 .../HyracksClientInterfaceRemoteProxy.java      |  130 ++
 .../hyracks/api/client/HyracksConnection.java   |  189 +++
 .../api/client/IHyracksClientConnection.java    |  187 +++
 .../api/client/IHyracksClientInterface.java     |   55 +
 .../hyracks/api/client/NodeControllerInfo.java  |   55 +
 .../apache/hyracks/api/client/NodeStatus.java   |   20 +
 .../impl/ActivityClusterGraphBuilder.java       |  176 +++
 .../impl/IConnectorDescriptorVisitor.java       |   22 +
 .../client/impl/IOperatorDescriptorVisitor.java |   22 +
 .../client/impl/JobActivityGraphBuilder.java    |  141 ++
 ...ionActivityClusterGraphGeneratorFactory.java |  113 ++
 .../hyracks/api/client/impl/PlanUtils.java      |   47 +
 .../apache/hyracks/api/comm/FixedSizeFrame.java |   59 +
 .../apache/hyracks/api/comm/FrameConstants.java |   48 +
 .../apache/hyracks/api/comm/FrameHelper.java    |   78 ++
 .../org/apache/hyracks/api/comm/IFrame.java     |   60 +
 .../apache/hyracks/api/comm/IFrameAppender.java |   55 +
 .../hyracks/api/comm/IFrameFieldAppender.java   |   47 +
 .../apache/hyracks/api/comm/IFrameReader.java   |   25 +
 .../hyracks/api/comm/IFrameTupleAccessor.java   |   44 +
 .../hyracks/api/comm/IFrameTupleAppender.java   |   39 +
 .../apache/hyracks/api/comm/IFrameWriter.java   |   79 ++
 .../hyracks/api/comm/IPartitionCollector.java   |   42 +
 .../api/comm/IPartitionWriterFactory.java       |   21 +
 .../apache/hyracks/api/comm/NetworkAddress.java |   98 ++
 .../hyracks/api/comm/NoShrinkVSizeFrame.java    |   31 +
 .../hyracks/api/comm/PartitionChannel.java      |   37 +
 .../org/apache/hyracks/api/comm/VSizeFrame.java |   76 +
 .../hyracks/api/constraints/Constraint.java     |   46 +
 .../api/constraints/IConstraintAcceptor.java    |   19 +
 .../constraints/PartitionConstraintHelper.java  |   44 +
 .../expressions/ConstantExpression.java         |   45 +
 .../expressions/ConstraintExpression.java       |   41 +
 .../expressions/LValueConstraintExpression.java |   19 +
 .../expressions/PartitionCountExpression.java   |   72 +
 .../PartitionLocationExpression.java            |   81 ++
 .../apache/hyracks/api/context/ICCContext.java  |   30 +
 .../api/context/IHyracksCommonContext.java      |   22 +
 .../api/context/IHyracksFrameMgrContext.java    |   39 +
 .../api/context/IHyracksJobletContext.java      |   35 +
 .../api/context/IHyracksRootContext.java        |   26 +
 .../api/context/IHyracksTaskContext.java        |   36 +
 .../apache/hyracks/api/dataflow/ActivityId.java |   94 ++
 .../api/dataflow/ConnectorDescriptorId.java     |   78 ++
 .../apache/hyracks/api/dataflow/IActivity.java  |   28 +
 .../api/dataflow/IActivityGraphBuilder.java     |   25 +
 .../api/dataflow/IConnectorDescriptor.java      |  135 ++
 .../hyracks/api/dataflow/IDataReader.java       |   39 +
 .../hyracks/api/dataflow/IDataWriter.java       |   48 +
 .../api/dataflow/IOpenableDataReader.java       |   21 +
 .../api/dataflow/IOpenableDataWriter.java       |   21 +
 .../api/dataflow/IOperatorDescriptor.java       |   93 ++
 .../api/dataflow/IOperatorNodePullable.java     |   19 +
 .../api/dataflow/IOperatorNodePushable.java     |   34 +
 .../api/dataflow/OperatorDescriptorId.java      |   85 ++
 .../api/dataflow/OperatorInstanceId.java        |   65 +
 .../hyracks/api/dataflow/TaskAttemptId.java     |   93 ++
 .../org/apache/hyracks/api/dataflow/TaskId.java |   93 ++
 .../connectors/ConnectorPolicyFactory.java      |   66 +
 .../dataflow/connectors/IConnectorPolicy.java   |   27 +
 .../IConnectorPolicyAssignmentPolicy.java       |   24 +
 .../connectors/PipeliningConnectorPolicy.java   |   39 +
 ...SideMaterializedBlockingConnectorPolicy.java |   39 +
 ...deMaterializedPipeliningConnectorPolicy.java |   39 +
 ...SideMaterializedBlockingConnectorPolicy.java |   39 +
 ...deMaterializedPipeliningConnectorPolicy.java |   39 +
 ...SideMaterializedBlockingConnectorPolicy.java |   40 +
 .../api/dataflow/state/IStateObject.java        |   33 +
 .../value/BinaryComparatorConstant.java         |   30 +
 .../api/dataflow/value/IBinaryComparator.java   |   21 +
 .../value/IBinaryComparatorFactory.java         |   21 +
 .../api/dataflow/value/IBinaryHashFunction.java |   21 +
 .../value/IBinaryHashFunctionFactory.java       |   21 +
 .../value/IBinaryHashFunctionFamily.java        |   21 +
 .../hyracks/api/dataflow/value/IComparator.java |   19 +
 .../api/dataflow/value/IComparatorFactory.java  |   21 +
 .../api/dataflow/value/IHashFunction.java       |   21 +
 .../dataflow/value/IHashFunctionFactory.java    |   21 +
 .../dataflow/value/ILinearizeComparator.java    |   20 +
 .../value/ILinearizeComparatorFactory.java      |   19 +
 .../dataflow/value/INormalizedKeyComputer.java  |   19 +
 .../value/INormalizedKeyComputerFactory.java    |   21 +
 .../hyracks/api/dataflow/value/INullWriter.java |   23 +
 .../api/dataflow/value/INullWriterFactory.java  |   21 +
 .../api/dataflow/value/IPredicateEvaluator.java |   25 +
 .../value/IPredicateEvaluatorFactory.java       |   26 +
 .../IPredicateEvaluatorFactoryProvider.java     |   26 +
 .../value/IRecordDescriptorProvider.java        |   23 +
 .../api/dataflow/value/IResultSerializer.java   |   38 +
 .../value/IResultSerializerFactory.java         |   29 +
 .../dataflow/value/ISerializerDeserializer.java |   48 +
 .../dataflow/value/ITuplePairComparator.java    |   25 +
 .../value/ITuplePairComparatorFactory.java      |   23 +
 .../dataflow/value/ITuplePartitionComputer.java |   22 +
 .../value/ITuplePartitionComputerFactory.java   |   21 +
 .../value/ITuplePartitionComputerFamily.java    |   21 +
 .../hyracks/api/dataflow/value/ITypeTraits.java |   23 +
 .../api/dataflow/value/JSONSerializable.java    |   27 +
 .../api/dataflow/value/RecordDescriptor.java    |   49 +
 .../api/dataset/DatasetDirectoryRecord.java     |  100 ++
 .../hyracks/api/dataset/DatasetJobRecord.java   |   68 +
 .../dataset/IDatasetInputChannelMonitor.java    |   27 +
 .../hyracks/api/dataset/IDatasetManager.java    |   28 +
 .../api/dataset/IDatasetPartitionManager.java   |   45 +
 .../api/dataset/IDatasetStateRecord.java        |   19 +
 .../hyracks/api/dataset/IHyracksDataset.java    |   22 +
 ...yracksDatasetDirectoryServiceConnection.java |   47 +
 ...HyracksDatasetDirectoryServiceInterface.java |   47 +
 .../api/dataset/IHyracksDatasetReader.java      |   25 +
 .../apache/hyracks/api/dataset/ResultSetId.java |   52 +
 .../hyracks/api/dataset/ResultSetMetaData.java  |   34 +
 .../hyracks/api/deployment/DeploymentId.java    |   78 ++
 .../api/exceptions/HyracksDataException.java    |   44 +
 .../api/exceptions/HyracksException.java        |   36 +
 .../apache/hyracks/api/io/FileReference.java    |   65 +
 .../org/apache/hyracks/api/io/IFileHandle.java  |   22 +
 .../org/apache/hyracks/api/io/IIOFuture.java    |   23 +
 .../org/apache/hyracks/api/io/IIOManager.java   |   53 +
 .../apache/hyracks/api/io/IODeviceHandle.java   |   47 +
 .../hyracks/api/io/IWorkspaceFileFactory.java   |   23 +
 .../org/apache/hyracks/api/io/IWritable.java    |   30 +
 .../apache/hyracks/api/job/ActivityCluster.java |  232 ++++
 .../hyracks/api/job/ActivityClusterGraph.java   |  172 +++
 .../hyracks/api/job/ActivityClusterId.java      |   77 ++
 .../api/job/IActivityClusterGraphGenerator.java |   25 +
 .../IActivityClusterGraphGeneratorFactory.java  |   28 +
 .../api/job/IConnectorDescriptorRegistry.java   |   22 +
 .../hyracks/api/job/IGlobalJobDataFactory.java  |   23 +
 .../hyracks/api/job/IJobLifecycleListener.java  |   25 +
 .../api/job/IJobSerializerDeserializer.java     |   70 +
 .../IJobSerializerDeserializerContainer.java    |   45 +
 .../hyracks/api/job/IJobletEventListener.java   |   21 +
 .../api/job/IJobletEventListenerFactory.java    |   23 +
 .../api/job/IOperatorDescriptorRegistry.java    |   22 +
 .../hyracks/api/job/IOperatorEnvironment.java   |   23 +
 .../hyracks/api/job/JobActivityGraph.java       |  105 ++
 .../org/apache/hyracks/api/job/JobFlag.java     |   19 +
 .../java/org/apache/hyracks/api/job/JobId.java  |   84 ++
 .../org/apache/hyracks/api/job/JobInfo.java     |   50 +
 .../api/job/JobSerializerDeserializer.java      |   64 +
 .../job/JobSerializerDeserializerContainer.java |   52 +
 .../hyracks/api/job/JobSpecification.java       |  409 ++++++
 .../org/apache/hyracks/api/job/JobStatus.java   |   22 +
 .../api/job/profiling/counters/ICounter.java    |   49 +
 .../job/profiling/counters/ICounterContext.java |   35 +
 .../api/lifecycle/ILifeCycleComponent.java      |   27 +
 .../lifecycle/ILifeCycleComponentManager.java   |   35 +
 .../lifecycle/LifeCycleComponentManager.java    |  169 +++
 .../apache/hyracks/api/messages/IMessage.java   |   24 +
 .../hyracks/api/messages/IMessageBroker.java    |   24 +
 .../hyracks/api/partitions/IPartition.java      |   27 +
 .../hyracks/api/partitions/PartitionId.java     |  127 ++
 .../api/partitions/ResultSetPartitionId.java    |   87 ++
 .../api/replication/IIOReplicationManager.java  |   27 +
 .../api/replication/IReplicationJob.java        |   45 +
 .../impl/AbstractReplicationJob.java            |   54 +
 .../hyracks/api/resources/IDeallocatable.java   |   19 +
 .../api/resources/IDeallocatableRegistry.java   |   19 +
 .../api/resources/memory/IMemoryManager.java    |   25 +
 .../rewriter/ActivityClusterGraphRewriter.java  |  381 +++++
 .../OneToOneConnectedActivityCluster.java       |  122 ++
 .../api/rewriter/runtime/SuperActivity.java     |  177 +++
 .../SuperActivityOperatorNodePushable.java      |  196 +++
 .../hyracks/api/topology/ClusterTopology.java   |   36 +
 .../hyracks/api/topology/NetworkEndpoint.java   |   46 +
 .../hyracks/api/topology/NetworkSwitch.java     |  108 ++
 .../hyracks/api/topology/NetworkTerminal.java   |   30 +
 .../api/topology/TopologyDefinitionParser.java  |  135 ++
 .../api/util/JavaSerializationUtils.java        |  117 ++
 .../client/dataset/DatasetClientContext.java    |   34 -
 .../hyracks/client/dataset/HyracksDataset.java  |   56 -
 ...yracksDatasetDirectoryServiceConnection.java |   53 -
 ...setDirectoryServiceInterfaceRemoteProxy.java |   51 -
 .../client/dataset/HyracksDatasetReader.java    |  258 ----
 .../client/net/ClientNetworkManager.java        |   56 -
 .../hyracks/client/stats/AggregateCounter.java  |   58 -
 .../uci/ics/hyracks/client/stats/Counters.java  |   34 -
 .../client/stats/IClusterCounterContext.java    |   25 -
 .../client/stats/impl/ClientCounterContext.java |  274 ----
 .../client/dataset/DatasetClientContext.java    |   34 +
 .../hyracks/client/dataset/HyracksDataset.java  |   56 +
 ...yracksDatasetDirectoryServiceConnection.java |   53 +
 ...setDirectoryServiceInterfaceRemoteProxy.java |   51 +
 .../client/dataset/HyracksDatasetReader.java    |  258 ++++
 .../client/net/ClientNetworkManager.java        |   56 +
 .../hyracks/client/stats/AggregateCounter.java  |   58 +
 .../apache/hyracks/client/stats/Counters.java   |   34 +
 .../client/stats/IClusterCounterContext.java    |   25 +
 .../client/stats/impl/ClientCounterContext.java |  274 ++++
 .../client/stats/ClientCounterContextTest.java  |   52 -
 .../ics/hyracks/client/stats/HyracksUtils.java  |   99 --
 .../client/stats/ClientCounterContextTest.java  |   52 +
 .../hyracks/client/stats/HyracksUtils.java      |   99 ++
 .../channels/DatasetNetworkInputChannel.java    |  150 --
 .../channels/IChannelConnectionFactory.java     |   24 -
 .../comm/channels/NetworkInputChannel.java      |  141 --
 .../comm/channels/NetworkOutputChannel.java     |  116 --
 .../comm/channels/ReadBufferFactory.java        |   51 -
 .../channels/DatasetNetworkInputChannel.java    |  150 ++
 .../channels/IChannelConnectionFactory.java     |   24 +
 .../comm/channels/NetworkInputChannel.java      |  141 ++
 .../comm/channels/NetworkOutputChannel.java     |  116 ++
 .../comm/channels/ReadBufferFactory.java        |   51 +
 .../uci/ics/hyracks/control/cc/CCDriver.java    |   43 -
 .../control/cc/ClusterControllerService.java    |  637 ---------
 .../hyracks/control/cc/NodeControllerState.java |  320 -----
 .../HyracksAdminConsoleApplication.java         |   39 -
 .../cc/adminconsole/pages/AbstractPage.java     |   27 -
 .../cc/adminconsole/pages/IndexPage.java        |   86 --
 .../cc/adminconsole/pages/JobDetailsPage.java   |  330 -----
 .../cc/adminconsole/pages/NodeDetailsPage.java  |   24 -
 .../cc/application/CCApplicationContext.java    |  106 --
 .../cc/dataset/DatasetDirectoryService.java     |  295 ----
 .../cc/dataset/IDatasetDirectoryService.java    |   46 -
 .../control/cc/job/ActivityClusterPlan.java     |   38 -
 .../hyracks/control/cc/job/ActivityPlan.java    |   39 -
 .../cc/job/IJobStatusConditionVariable.java     |   19 -
 .../uci/ics/hyracks/control/cc/job/JobRun.java  |  399 ------
 .../uci/ics/hyracks/control/cc/job/Task.java    |   68 -
 .../ics/hyracks/control/cc/job/TaskAttempt.java |  102 --
 .../ics/hyracks/control/cc/job/TaskCluster.java |   90 --
 .../control/cc/job/TaskClusterAttempt.java      |  101 --
 .../hyracks/control/cc/job/TaskClusterId.java   |   73 -
 .../cc/partitions/PartitionMatchMaker.java      |  207 ---
 .../control/cc/partitions/PartitionUtils.java   |   39 -
 .../cc/scheduler/ActivityClusterPlanner.java    |  444 ------
 .../cc/scheduler/ActivityPartitionDetails.java  |   49 -
 .../control/cc/scheduler/JobScheduler.java      |  748 ----------
 .../cc/scheduler/PartitionConstraintSolver.java |  125 --
 .../cc/scheduler/RankedRunnableTaskCluster.java |   47 -
 .../control/cc/scheduler/Runnability.java       |  101 --
 .../cc/web/ApplicationInstallationHandler.java  |  121 --
 .../control/cc/web/JobsRESTAPIFunction.java     |   66 -
 .../control/cc/web/NodesRESTAPIFunction.java    |   50 -
 .../cc/web/StateDumpRESTAPIFunction.java        |   47 -
 .../ics/hyracks/control/cc/web/WebServer.java   |  124 --
 .../cc/web/util/IJSONOutputFunction.java        |   21 -
 .../cc/web/util/JSONOutputRequestHandler.java   |   53 -
 .../hyracks/control/cc/web/util/JSONUtils.java  |   32 -
 .../control/cc/web/util/RoutingHandler.java     |   65 -
 .../control/cc/work/AbstractHeartbeatWork.java  |   49 -
 .../cc/work/AbstractTaskLifecycleWork.java      |   75 -
 .../control/cc/work/ApplicationMessageWork.java |   66 -
 .../control/cc/work/CliDeployBinaryWork.java    |  108 --
 .../control/cc/work/CliUnDeployBinaryWork.java  |  104 --
 .../control/cc/work/ClusterShutdownWork.java    |  102 --
 .../control/cc/work/GatherStateDumpsWork.java   |   98 --
 .../work/GetActivityClusterGraphJSONWork.java   |   50 -
 .../GetDatasetDirectoryServiceInfoWork.java     |   41 -
 .../cc/work/GetIpAddressNodeNameMapWork.java    |   37 -
 .../hyracks/control/cc/work/GetJobInfoWork.java |   49 -
 .../control/cc/work/GetJobRunJSONWork.java      |   50 -
 .../control/cc/work/GetJobStatusWork.java       |   48 -
 .../cc/work/GetJobSummariesJSONWork.java        |   58 -
 .../cc/work/GetNodeControllersInfoWork.java     |   47 -
 .../control/cc/work/GetNodeDetailsJSONWork.java |   46 -
 .../cc/work/GetNodeSummariesJSONWork.java       |   42 -
 .../work/GetResultPartitionLocationsWork.java   |   67 -
 .../control/cc/work/GetResultStatusWork.java    |   56 -
 .../hyracks/control/cc/work/JobCleanupWork.java |  138 --
 .../hyracks/control/cc/work/JobStartWork.java   |   74 -
 .../cc/work/JobletCleanupNotificationWork.java  |   72 -
 .../control/cc/work/NodeHeartbeatWork.java      |   37 -
 .../control/cc/work/NotifyDeployBinaryWork.java |   52 -
 .../control/cc/work/NotifyShutdownWork.java     |   47 -
 .../cc/work/NotifyStateDumpResponse.java        |   41 -
 .../control/cc/work/RegisterNodeWork.java       |   90 --
 .../work/RegisterPartitionAvailibilityWork.java |   62 -
 .../cc/work/RegisterPartitionRequestWork.java   |   59 -
 .../RegisterResultPartitionLocationWork.java    |   63 -
 .../control/cc/work/RemoveDeadNodesWork.java    |   77 --
 .../control/cc/work/ReportProfilesWork.java     |   52 -
 .../work/ReportResultPartitionFailureWork.java  |   47 -
 ...eportResultPartitionWriteCompletionWork.java |   48 -
 .../control/cc/work/TaskCompleteWork.java       |   62 -
 .../control/cc/work/TaskFailureWork.java        |   47 -
 .../control/cc/work/UnregisterNodeWork.java     |   37 -
 .../cc/work/WaitForJobCompletionWork.java       |   80 --
 .../org/apache/hyracks/control/cc/CCDriver.java |   43 +
 .../control/cc/ClusterControllerService.java    |  637 +++++++++
 .../hyracks/control/cc/NodeControllerState.java |  320 +++++
 .../HyracksAdminConsoleApplication.java         |   39 +
 .../cc/adminconsole/pages/AbstractPage.java     |   27 +
 .../cc/adminconsole/pages/IndexPage.java        |   86 ++
 .../cc/adminconsole/pages/JobDetailsPage.java   |  330 +++++
 .../cc/adminconsole/pages/NodeDetailsPage.java  |   24 +
 .../cc/application/CCApplicationContext.java    |  106 ++
 .../cc/dataset/DatasetDirectoryService.java     |  295 ++++
 .../cc/dataset/IDatasetDirectoryService.java    |   46 +
 .../control/cc/job/ActivityClusterPlan.java     |   38 +
 .../hyracks/control/cc/job/ActivityPlan.java    |   39 +
 .../cc/job/IJobStatusConditionVariable.java     |   19 +
 .../apache/hyracks/control/cc/job/JobRun.java   |  399 ++++++
 .../org/apache/hyracks/control/cc/job/Task.java |   68 +
 .../hyracks/control/cc/job/TaskAttempt.java     |  102 ++
 .../hyracks/control/cc/job/TaskCluster.java     |   90 ++
 .../control/cc/job/TaskClusterAttempt.java      |  101 ++
 .../hyracks/control/cc/job/TaskClusterId.java   |   73 +
 .../cc/partitions/PartitionMatchMaker.java      |  207 +++
 .../control/cc/partitions/PartitionUtils.java   |   39 +
 .../cc/scheduler/ActivityClusterPlanner.java    |  444 ++++++
 .../cc/scheduler/ActivityPartitionDetails.java  |   49 +
 .../control/cc/scheduler/JobScheduler.java      |  748 ++++++++++
 .../cc/scheduler/PartitionConstraintSolver.java |  125 ++
 .../cc/scheduler/RankedRunnableTaskCluster.java |   47 +
 .../control/cc/scheduler/Runnability.java       |  101 ++
 .../cc/web/ApplicationInstallationHandler.java  |  121 ++
 .../control/cc/web/JobsRESTAPIFunction.java     |   66 +
 .../control/cc/web/NodesRESTAPIFunction.java    |   50 +
 .../cc/web/StateDumpRESTAPIFunction.java        |   47 +
 .../hyracks/control/cc/web/WebServer.java       |  124 ++
 .../cc/web/util/IJSONOutputFunction.java        |   21 +
 .../cc/web/util/JSONOutputRequestHandler.java   |   53 +
 .../hyracks/control/cc/web/util/JSONUtils.java  |   32 +
 .../control/cc/web/util/RoutingHandler.java     |   65 +
 .../control/cc/work/AbstractHeartbeatWork.java  |   49 +
 .../cc/work/AbstractTaskLifecycleWork.java      |   75 +
 .../control/cc/work/ApplicationMessageWork.java |   66 +
 .../control/cc/work/CliDeployBinaryWork.java    |  108 ++
 .../control/cc/work/CliUnDeployBinaryWork.java  |  104 ++
 .../control/cc/work/ClusterShutdownWork.java    |  102 ++
 .../control/cc/work/GatherStateDumpsWork.java   |   98 ++
 .../work/GetActivityClusterGraphJSONWork.java   |   50 +
 .../GetDatasetDirectoryServiceInfoWork.java     |   41 +
 .../cc/work/GetIpAddressNodeNameMapWork.java    |   37 +
 .../hyracks/control/cc/work/GetJobInfoWork.java |   49 +
 .../control/cc/work/GetJobRunJSONWork.java      |   50 +
 .../control/cc/work/GetJobStatusWork.java       |   48 +
 .../cc/work/GetJobSummariesJSONWork.java        |   58 +
 .../cc/work/GetNodeControllersInfoWork.java     |   47 +
 .../control/cc/work/GetNodeDetailsJSONWork.java |   46 +
 .../cc/work/GetNodeSummariesJSONWork.java       |   42 +
 .../work/GetResultPartitionLocationsWork.java   |   67 +
 .../control/cc/work/GetResultStatusWork.java    |   56 +
 .../hyracks/control/cc/work/JobCleanupWork.java |  138 ++
 .../hyracks/control/cc/work/JobStartWork.java   |   74 +
 .../cc/work/JobletCleanupNotificationWork.java  |   72 +
 .../control/cc/work/NodeHeartbeatWork.java      |   37 +
 .../control/cc/work/NotifyDeployBinaryWork.java |   52 +
 .../control/cc/work/NotifyShutdownWork.java     |   47 +
 .../cc/work/NotifyStateDumpResponse.java        |   41 +
 .../control/cc/work/RegisterNodeWork.java       |   90 ++
 .../work/RegisterPartitionAvailibilityWork.java |   62 +
 .../cc/work/RegisterPartitionRequestWork.java   |   59 +
 .../RegisterResultPartitionLocationWork.java    |   63 +
 .../control/cc/work/RemoveDeadNodesWork.java    |   77 ++
 .../control/cc/work/ReportProfilesWork.java     |   52 +
 .../work/ReportResultPartitionFailureWork.java  |   47 +
 ...eportResultPartitionWriteCompletionWork.java |   48 +
 .../control/cc/work/TaskCompleteWork.java       |   62 +
 .../control/cc/work/TaskFailureWork.java        |   47 +
 .../control/cc/work/UnregisterNodeWork.java     |   37 +
 .../cc/work/WaitForJobCompletionWork.java       |   80 ++
 .../control/common/AbstractRemoteService.java   |   22 -
 .../common/application/ApplicationContext.java  |   73 -
 .../common/application/ApplicationStatus.java   |   23 -
 .../control/common/base/IClusterController.java |   69 -
 .../control/common/base/INodeController.java    |   51 -
 .../control/common/context/ServerContext.java   |   40 -
 .../control/common/controllers/CCConfig.java    |  116 --
 .../control/common/controllers/NCConfig.java    |  188 ---
 .../common/controllers/NodeParameters.java      |   63 -
 .../common/controllers/NodeRegistration.java    |  159 ---
 .../common/dataset/ResultStateSweeper.java      |   78 --
 .../ClassLoaderJobSerializerDeserializer.java   |  106 --
 .../common/deployment/DeploymentRun.java        |   62 -
 .../common/deployment/DeploymentStatus.java     |   21 -
 .../common/deployment/DeploymentUtils.java      |  225 ---
 .../IDeploymentStatusConditionVariable.java     |   32 -
 .../control/common/heartbeat/HeartbeatData.java |  130 --
 .../common/heartbeat/HeartbeatSchema.java       |   45 -
 .../control/common/ipc/CCNCFunctions.java       | 1309 ------------------
 .../ipc/ClusterControllerRemoteProxy.java       |  153 --
 .../common/ipc/NodeControllerRemoteProxy.java   |   93 --
 .../control/common/job/PartitionDescriptor.java |   71 -
 .../control/common/job/PartitionRequest.java    |   61 -
 .../control/common/job/PartitionState.java      |   31 -
 .../common/job/TaskAttemptDescriptor.java       |  157 ---
 .../common/job/profiling/counters/Counter.java  |   51 -
 .../counters/MultiResolutionEventProfiler.java  |  121 --
 .../job/profiling/om/AbstractProfile.java       |   80 --
 .../common/job/profiling/om/JobProfile.java     |  107 --
 .../common/job/profiling/om/JobletProfile.java  |  109 --
 .../job/profiling/om/PartitionProfile.java      |   85 --
 .../common/job/profiling/om/TaskProfile.java    |  125 --
 .../hyracks/control/common/logs/LogFile.java    |   47 -
 .../control/common/service/AbstractService.java |   18 -
 .../control/common/service/IService.java        |   21 -
 .../IShutdownStatusConditionVariable.java       |   24 -
 .../control/common/shutdown/ShutdownRun.java    |   58 -
 .../control/common/utils/ExceptionUtils.java    |   60 -
 .../control/common/work/AbstractWork.java       |   34 -
 .../control/common/work/FutureValue.java        |   62 -
 .../control/common/work/IPCResponder.java       |   47 -
 .../control/common/work/IResultCallback.java    |   21 -
 .../control/common/work/SynchronizableWork.java |   55 -
 .../hyracks/control/common/work/WorkQueue.java  |  132 --
 .../control/common/AbstractRemoteService.java   |   22 +
 .../common/application/ApplicationContext.java  |   73 +
 .../common/application/ApplicationStatus.java   |   23 +
 .../control/common/base/IClusterController.java |   69 +
 .../control/common/base/INodeController.java    |   51 +
 .../control/common/context/ServerContext.java   |   40 +
 .../control/common/controllers/CCConfig.java    |  116 ++
 .../control/common/controllers/NCConfig.java    |  188 +++
 .../common/controllers/NodeParameters.java      |   63 +
 .../common/controllers/NodeRegistration.java    |  159 +++
 .../common/dataset/ResultStateSweeper.java      |   78 ++
 .../ClassLoaderJobSerializerDeserializer.java   |  106 ++
 .../common/deployment/DeploymentRun.java        |   62 +
 .../common/deployment/DeploymentStatus.java     |   21 +
 .../common/deployment/DeploymentUtils.java      |  225 +++
 .../IDeploymentStatusConditionVariable.java     |   32 +
 .../control/common/heartbeat/HeartbeatData.java |  130 ++
 .../common/heartbeat/HeartbeatSchema.java       |   45 +
 .../control/common/ipc/CCNCFunctions.java       | 1309 ++++++++++++++++++
 .../ipc/ClusterControllerRemoteProxy.java       |  153 ++
 .../common/ipc/NodeControllerRemoteProxy.java   |   93 ++
 .../control/common/job/PartitionDescriptor.java |   71 +
 .../control/common/job/PartitionRequest.java    |   61 +
 .../control/common/job/PartitionState.java      |   31 +
 .../common/job/TaskAttemptDescriptor.java       |  157 +++
 .../common/job/profiling/counters/Counter.java  |   51 +
 .../counters/MultiResolutionEventProfiler.java  |  121 ++
 .../job/profiling/om/AbstractProfile.java       |   80 ++
 .../common/job/profiling/om/JobProfile.java     |  107 ++
 .../common/job/profiling/om/JobletProfile.java  |  109 ++
 .../job/profiling/om/PartitionProfile.java      |   85 ++
 .../common/job/profiling/om/TaskProfile.java    |  125 ++
 .../hyracks/control/common/logs/LogFile.java    |   47 +
 .../control/common/service/AbstractService.java |   18 +
 .../control/common/service/IService.java        |   21 +
 .../IShutdownStatusConditionVariable.java       |   24 +
 .../control/common/shutdown/ShutdownRun.java    |   58 +
 .../control/common/utils/ExceptionUtils.java    |   60 +
 .../control/common/work/AbstractWork.java       |   34 +
 .../control/common/work/FutureValue.java        |   62 +
 .../control/common/work/IPCResponder.java       |   47 +
 .../control/common/work/IResultCallback.java    |   21 +
 .../control/common/work/SynchronizableWork.java |   55 +
 .../hyracks/control/common/work/WorkQueue.java  |  132 ++
 .../edu/uci/ics/hyracks/control/nc/Joblet.java  |  341 -----
 .../uci/ics/hyracks/control/nc/NCDriver.java    |   63 -
 .../control/nc/NodeControllerService.java       |  583 --------
 .../edu/uci/ics/hyracks/control/nc/Task.java    |  379 -----
 .../nc/application/NCApplicationContext.java    |   96 --
 .../nc/dataset/DatasetMemoryManager.java        |  241 ----
 .../nc/dataset/DatasetPartitionManager.java     |  268 ----
 .../nc/dataset/DatasetPartitionReader.java      |   97 --
 .../nc/dataset/DatasetPartitionWriter.java      |  139 --
 .../ics/hyracks/control/nc/dataset/Page.java    |   33 -
 .../hyracks/control/nc/dataset/ResultState.java |  319 -----
 .../ics/hyracks/control/nc/io/FileHandle.java   |   90 --
 .../ics/hyracks/control/nc/io/IOManager.java    |  255 ----
 .../control/nc/io/WorkspaceFileFactory.java     |   48 -
 .../control/nc/io/profiling/IIOCounter.java     |   29 -
 .../nc/io/profiling/IOCounterDefault.java       |   30 -
 .../nc/io/profiling/IOCounterFactory.java       |   35 -
 .../control/nc/io/profiling/IOCounterLinux.java |  124 --
 .../control/nc/io/profiling/IOCounterOSX.java   |   30 -
 .../control/nc/net/DatasetNetworkManager.java   |  160 ---
 .../hyracks/control/nc/net/NetworkManager.java  |  160 ---
 .../nc/partitions/MaterializedPartition.java    |  105 --
 .../MaterializedPartitionInputChannel.java      |  137 --
 .../partitions/MaterializedPartitionWriter.java |  102 --
 .../MaterializingPipelinedPartition.java        |  194 ---
 .../control/nc/partitions/PartitionManager.java |  151 --
 .../nc/partitions/PipelinedPartition.java       |  114 --
 .../ReceiveSideMaterializingCollector.java      |  178 ---
 .../ConnectorReceiverProfilingFrameReader.java  |   59 -
 .../ConnectorSenderProfilingFrameWriter.java    |   65 -
 .../ProfilingPartitionWriterFactory.java        |   85 --
 .../resources/DefaultDeallocatableRegistry.java |   44 -
 .../nc/resources/memory/FrameManager.java       |   85 --
 .../nc/resources/memory/MemoryManager.java      |   55 -
 .../control/nc/runtime/RootHyracksContext.java  |   43 -
 .../hyracks/control/nc/work/AbortTasksWork.java |   67 -
 .../control/nc/work/ApplicationMessageWork.java |   64 -
 .../control/nc/work/BuildJobProfilesWork.java   |   57 -
 .../control/nc/work/CleanupJobletWork.java      |   66 -
 .../control/nc/work/DeployBinaryWork.java       |   65 -
 .../control/nc/work/NotifyTaskCompleteWork.java |   43 -
 .../control/nc/work/NotifyTaskFailureWork.java  |   52 -
 .../work/ReportPartitionAvailabilityWork.java   |   58 -
 .../hyracks/control/nc/work/ShutdownWork.java   |   63 -
 .../hyracks/control/nc/work/StartTasksWork.java |  276 ----
 .../hyracks/control/nc/work/StateDumpWork.java  |   39 -
 .../control/nc/work/UnDeployBinaryWork.java     |   58 -
 .../org/apache/hyracks/control/nc/Joblet.java   |  341 +++++
 .../org/apache/hyracks/control/nc/NCDriver.java |   63 +
 .../control/nc/NodeControllerService.java       |  583 ++++++++
 .../org/apache/hyracks/control/nc/Task.java     |  379 +++++
 .../nc/application/NCApplicationContext.java    |   96 ++
 .../nc/dataset/DatasetMemoryManager.java        |  241 ++++
 .../nc/dataset/DatasetPartitionManager.java     |  268 ++++
 .../nc/dataset/DatasetPartitionReader.java      |   97 ++
 .../nc/dataset/DatasetPartitionWriter.java      |  139 ++
 .../apache/hyracks/control/nc/dataset/Page.java |   33 +
 .../hyracks/control/nc/dataset/ResultState.java |  319 +++++
 .../hyracks/control/nc/io/FileHandle.java       |   90 ++
 .../apache/hyracks/control/nc/io/IOManager.java |  255 ++++
 .../control/nc/io/WorkspaceFileFactory.java     |   48 +
 .../control/nc/io/profiling/IIOCounter.java     |   29 +
 .../nc/io/profiling/IOCounterDefault.java       |   30 +
 .../nc/io/profiling/IOCounterFactory.java       |   35 +
 .../control/nc/io/profiling/IOCounterLinux.java |  124 ++
 .../control/nc/io/profiling/IOCounterOSX.java   |   30 +
 .../control/nc/net/DatasetNetworkManager.java   |  160 +++
 .../hyracks/control/nc/net/NetworkManager.java  |  160 +++
 .../nc/partitions/MaterializedPartition.java    |  105 ++
 .../MaterializedPartitionInputChannel.java      |  137 ++
 .../partitions/MaterializedPartitionWriter.java |  102 ++
 .../MaterializingPipelinedPartition.java        |  194 +++
 .../control/nc/partitions/PartitionManager.java |  151 ++
 .../nc/partitions/PipelinedPartition.java       |  114 ++
 .../ReceiveSideMaterializingCollector.java      |  178 +++
 .../ConnectorReceiverProfilingFrameReader.java  |   59 +
 .../ConnectorSenderProfilingFrameWriter.java    |   65 +
 .../ProfilingPartitionWriterFactory.java        |   85 ++
 .../resources/DefaultDeallocatableRegistry.java |   44 +
 .../nc/resources/memory/FrameManager.java       |   85 ++
 .../nc/resources/memory/MemoryManager.java      |   55 +
 .../control/nc/runtime/RootHyracksContext.java  |   43 +
 .../hyracks/control/nc/work/AbortTasksWork.java |   67 +
 .../control/nc/work/ApplicationMessageWork.java |   64 +
 .../control/nc/work/BuildJobProfilesWork.java   |   57 +
 .../control/nc/work/CleanupJobletWork.java      |   66 +
 .../control/nc/work/DeployBinaryWork.java       |   65 +
 .../control/nc/work/NotifyTaskCompleteWork.java |   43 +
 .../control/nc/work/NotifyTaskFailureWork.java  |   52 +
 .../work/ReportPartitionAvailabilityWork.java   |   58 +
 .../hyracks/control/nc/work/ShutdownWork.java   |   63 +
 .../hyracks/control/nc/work/StartTasksWork.java |  276 ++++
 .../hyracks/control/nc/work/StateDumpWork.java  |   39 +
 .../control/nc/work/UnDeployBinaryWork.java     |   58 +
 .../std/accessors/MurmurHash3BinaryHash.java    |   72 -
 .../MurmurHash3BinaryHashFunctionFamily.java    |   46 -
 .../PointableBinaryComparatorFactory.java       |   51 -
 .../PointableBinaryHashFunctionFactory.java     |   47 -
 .../UTF8StringBinaryHashFunctionFamily.java     |   53 -
 .../std/algorithms/BinarySearchAlgorithm.java   |   74 -
 .../hyracks/data/std/api/AbstractPointable.java |   50 -
 .../ics/hyracks/data/std/api/IComparable.java   |   21 -
 .../data/std/api/IDataOutputProvider.java       |   21 -
 .../uci/ics/hyracks/data/std/api/IHashable.java |   19 -
 .../data/std/api/IMutableValueStorage.java      |   19 -
 .../uci/ics/hyracks/data/std/api/INumeric.java  |   29 -
 .../ics/hyracks/data/std/api/IPointable.java    |   21 -
 .../hyracks/data/std/api/IPointableFactory.java |   25 -
 .../hyracks/data/std/api/IValueReference.java   |   23 -
 .../collections/api/IValueReferenceVector.java  |   58 -
 .../data/std/primitive/BooleanPointable.java    |   85 --
 .../data/std/primitive/ByteArrayPointable.java  |  101 --
 .../data/std/primitive/BytePointable.java       |  130 --
 .../data/std/primitive/DoublePointable.java     |  151 --
 .../data/std/primitive/FloatPointable.java      |  136 --
 .../data/std/primitive/IntegerPointable.java    |  134 --
 .../data/std/primitive/LongPointable.java       |  141 --
 .../std/primitive/RawUTF8StringPointable.java   |  105 --
 .../data/std/primitive/ShortPointable.java      |  131 --
 .../data/std/primitive/UTF8StringPointable.java |  219 ---
 .../data/std/primitive/UTF8StringWriter.java    |   75 -
 .../data/std/primitive/VoidPointable.java       |   50 -
 .../data/std/util/ArrayBackedValueStorage.java  |   64 -
 .../util/ByteArrayAccessibleOutputStream.java   |   76 -
 .../hyracks/data/std/util/GrowableArray.java    |   49 -
 .../std/accessors/MurmurHash3BinaryHash.java    |   72 +
 .../MurmurHash3BinaryHashFunctionFamily.java    |   46 +
 .../PointableBinaryComparatorFactory.java       |   51 +
 .../PointableBinaryHashFunctionFactory.java     |   47 +
 .../UTF8StringBinaryHashFunctionFamily.java     |   53 +
 .../std/algorithms/BinarySearchAlgorithm.java   |   74 +
 .../hyracks/data/std/api/AbstractPointable.java |   50 +
 .../hyracks/data/std/api/IComparable.java       |   21 +
 .../data/std/api/IDataOutputProvider.java       |   21 +
 .../apache/hyracks/data/std/api/IHashable.java  |   19 +
 .../data/std/api/IMutableValueStorage.java      |   19 +
 .../apache/hyracks/data/std/api/INumeric.java   |   29 +
 .../apache/hyracks/data/std/api/IPointable.java |   21 +
 .../hyracks/data/std/api/IPointableFactory.java |   25 +
 .../hyracks/data/std/api/IValueReference.java   |   23 +
 .../collections/api/IValueReferenceVector.java  |   58 +
 .../data/std/primitive/BooleanPointable.java    |   85 ++
 .../data/std/primitive/ByteArrayPointable.java  |  101 ++
 .../data/std/primitive/BytePointable.java       |  130 ++
 .../data/std/primitive/DoublePointable.java     |  151 ++
 .../data/std/primitive/FloatPointable.java      |  136 ++
 .../data/std/primitive/IntegerPointable.java    |  134 ++
 .../data/std/primitive/LongPointable.java       |  141 ++
 .../std/primitive/RawUTF8StringPointable.java   |  105 ++
 .../data/std/primitive/ShortPointable.java      |  131 ++
 .../data/std/primitive/UTF8StringPointable.java |  219 +++
 .../data/std/primitive/UTF8StringWriter.java    |   75 +
 .../data/std/primitive/VoidPointable.java       |   50 +
 .../data/std/util/ArrayBackedValueStorage.java  |   64 +
 .../util/ByteArrayAccessibleOutputStream.java   |   76 +
 .../hyracks/data/std/util/GrowableArray.java    |   49 +
 .../std/primitive/ByteArrayPointableTest.java   |   65 -
 .../std/primitive/ByteArrayPointableTest.java   |   65 +
 .../common/comm/io/AbstractFrameAppender.java   |   99 --
 .../common/comm/io/ArrayTupleBuilder.java       |  165 ---
 .../common/comm/io/ArrayTupleReference.java     |   55 -
 .../common/comm/io/FrameDeserializer.java       |   97 --
 .../comm/io/FrameDeserializingDataReader.java   |   75 -
 .../comm/io/FrameDeserializingDataWriter.java   |   57 -
 .../common/comm/io/FrameFixedFieldAppender.java |  108 --
 .../comm/io/FrameFixedFieldTupleAppender.java   |  144 --
 .../common/comm/io/FrameOutputStream.java       |   59 -
 .../common/comm/io/FrameTupleAccessor.java      |  160 ---
 .../common/comm/io/FrameTupleAppender.java      |  269 ----
 .../comm/io/FrameTupleAppenderAccessor.java     |  131 --
 .../comm/io/FrameTupleAppenderWrapper.java      |   95 --
 .../comm/io/FrameTuplePairComparator.java       |   58 -
 .../comm/io/ResultFrameTupleAccessor.java       |   41 -
 .../common/comm/io/SerializingDataWriter.java   |   87 --
 .../common/comm/util/ByteBufferInputStream.java |   52 -
 .../dataflow/common/comm/util/FrameUtils.java   |  283 ----
 .../FrameTupleFieldValueReference.java          |   48 -
 .../data/accessors/FrameTupleReference.java     |   58 -
 .../data/accessors/IFrameTupleReference.java    |   23 -
 .../common/data/accessors/ITupleReference.java  |   25 -
 .../BooleanSerializerDeserializer.java          |   49 -
 .../ByteArraySerializerDeserializer.java        |   75 -
 .../DoubleSerializerDeserializer.java           |   49 -
 .../FloatSerializerDeserializer.java            |   49 -
 .../IntArraySerializerDeserializer.java         |   57 -
 .../Integer64SerializerDeserializer.java        |   50 -
 .../IntegerSerializerDeserializer.java          |   49 -
 .../ShortSerializerDeserializer.java            |   49 -
 .../UTF8StringSerializerDeserializer.java       |   49 -
 .../ByteArrayNormalizedKeyComputerFactory.java  |   45 -
 .../DoubleNormalizedKeyComputerFactory.java     |   43 -
 .../FloatNormalizedKeyComputerFactory.java      |   43 -
 .../Integer64NormalizedKeyComputerFactory.java  |   70 -
 .../IntegerNormalizedKeyComputerFactory.java    |   35 -
 .../UTF8StringNormalizedKeyComputerFactory.java |   43 -
 .../parsers/ByteArrayBase64ParserFactory.java   |  243 ----
 .../data/parsers/ByteArrayHexParserFactory.java |  128 --
 .../data/parsers/DoubleParserFactory.java       |   46 -
 .../common/data/parsers/FloatParserFactory.java |   46 -
 .../common/data/parsers/IValueParser.java       |   23 -
 .../data/parsers/IValueParserFactory.java       |   21 -
 .../data/parsers/IntegerParserFactory.java      |  125 --
 .../common/data/parsers/LongParserFactory.java  |  126 --
 .../data/parsers/UTF8StringParserFactory.java   |   86 --
 .../FieldHashPartitionComputerFactory.java      |   65 -
 .../FieldHashPartitionComputerFamily.java       |   62 -
 .../RandomPartitionComputerFactory.java         |   49 -
 .../partition/RepartitionComputerFactory.java   |   44 -
 .../partition/RepartitionComputerFamily.java    |   47 -
 .../FieldRangePartitionComputerFactory.java     |   97 --
 .../common/data/partition/range/IRangeMap.java  |   31 -
 .../common/data/partition/range/RangeMap.java   |   94 --
 .../dataflow/common/data/util/StringUtils.java  |   43 -
 .../dataflow/common/io/RunFileReader.java       |   81 --
 .../dataflow/common/io/RunFileWriter.java       |   78 --
 .../dataflow/common/util/IntSerDeUtils.java     |   31 -
 .../dataflow/common/util/ReflectionUtils.java   |   29 -
 .../dataflow/common/util/SerdeUtils.java        |  143 --
 .../dataflow/common/util/TupleUtils.java        |  136 --
 .../common/comm/io/AbstractFrameAppender.java   |   99 ++
 .../common/comm/io/ArrayTupleBuilder.java       |  165 +++
 .../common/comm/io/ArrayTupleReference.java     |   55 +
 .../common/comm/io/FrameDeserializer.java       |   97 ++
 .../comm/io/FrameDeserializingDataReader.java   |   75 +
 .../comm/io/FrameDeserializingDataWriter.java   |   57 +
 .../common/comm/io/FrameFixedFieldAppender.java |  108 ++
 .../comm/io/FrameFixedFieldTupleAppender.java   |  144 ++
 .../common/comm/io/FrameOutputStream.java       |   59 +
 .../common/comm/io/FrameTupleAccessor.java      |  160 +++
 .../common/comm/io/FrameTupleAppender.java      |  269 ++++
 .../comm/io/FrameTupleAppenderAccessor.java     |  131 ++
 .../comm/io/FrameTupleAppenderWrapper.java      |   95 ++
 .../comm/io/FrameTuplePairComparator.java       |   58 +
 .../comm/io/ResultFrameTupleAccessor.java       |   41 +
 .../common/comm/io/SerializingDataWriter.java   |   87 ++
 .../common/comm/util/ByteBufferInputStream.java |   52 +
 .../dataflow/common/comm/util/FrameUtils.java   |  283 ++++
 .../FrameTupleFieldValueReference.java          |   48 +
 .../data/accessors/FrameTupleReference.java     |   58 +
 .../data/accessors/IFrameTupleReference.java    |   23 +
 .../common/data/accessors/ITupleReference.java  |   25 +
 .../BooleanSerializerDeserializer.java          |   49 +
 .../ByteArraySerializerDeserializer.java        |   75 +
 .../DoubleSerializerDeserializer.java           |   49 +
 .../FloatSerializerDeserializer.java            |   49 +
 .../IntArraySerializerDeserializer.java         |   57 +
 .../Integer64SerializerDeserializer.java        |   50 +
 .../IntegerSerializerDeserializer.java          |   49 +
 .../ShortSerializerDeserializer.java            |   49 +
 .../UTF8StringSerializerDeserializer.java       |   49 +
 .../ByteArrayNormalizedKeyComputerFactory.java  |   45 +
 .../DoubleNormalizedKeyComputerFactory.java     |   43 +
 .../FloatNormalizedKeyComputerFactory.java      |   43 +
 .../Integer64NormalizedKeyComputerFactory.java  |   70 +
 .../IntegerNormalizedKeyComputerFactory.java    |   35 +
 .../UTF8StringNormalizedKeyComputerFactory.java |   43 +
 .../parsers/ByteArrayBase64ParserFactory.java   |  243 ++++
 .../data/parsers/ByteArrayHexParserFactory.java |  128 ++
 .../data/parsers/DoubleParserFactory.java       |   46 +
 .../common/data/parsers/FloatParserFactory.java |   46 +
 .../common/data/parsers/IValueParser.java       |   23 +
 .../data/parsers/IValueParserFactory.java       |   21 +
 .../data/parsers/IntegerParserFactory.java      |  125 ++
 .../common/data/parsers/LongParserFactory.java  |  126 ++
 .../data/parsers/UTF8StringParserFactory.java   |   86 ++
 .../FieldHashPartitionComputerFactory.java      |   65 +
 .../FieldHashPartitionComputerFamily.java       |   62 +
 .../RandomPartitionComputerFactory.java         |   49 +
 .../partition/RepartitionComputerFactory.java   |   44 +
 .../partition/RepartitionComputerFamily.java    |   47 +
 .../FieldRangePartitionComputerFactory.java     |   97 ++
 .../common/data/partition/range/IRangeMap.java  |   31 +
 .../common/data/partition/range/RangeMap.java   |   94 ++
 .../dataflow/common/data/util/StringUtils.java  |   43 +
 .../dataflow/common/io/RunFileReader.java       |   81 ++
 .../dataflow/common/io/RunFileWriter.java       |   78 ++
 .../dataflow/common/util/IntSerDeUtils.java     |   31 +
 .../dataflow/common/util/ReflectionUtils.java   |   29 +
 .../dataflow/common/util/SerdeUtils.java        |  143 ++
 .../dataflow/common/util/TupleUtils.java        |  136 ++
 .../FrameFixedFieldTupleAppenderTest.java       |  215 ---
 .../ByteArraySerializerDeserializerTest.java    |   72 -
 ...teArrayNormalizedKeyComputerFactoryTest.java |   89 --
 .../ByteArrayBase64ParserFactoryTest.java       |   74 -
 .../parsers/ByteArrayHexParserFactoryTest.java  |   64 -
 .../FrameFixedFieldTupleAppenderTest.java       |  215 +++
 .../ByteArraySerializerDeserializerTest.java    |   72 +
 ...teArrayNormalizedKeyComputerFactoryTest.java |   89 ++
 .../ByteArrayBase64ParserFactoryTest.java       |   74 +
 .../parsers/ByteArrayHexParserFactoryTest.java  |   64 +
 .../AbstractHadoopOperatorDescriptor.java       |  168 ---
 .../hadoop/HadoopMapperOperatorDescriptor.java  |  446 ------
 .../hadoop/HadoopReadOperatorDescriptor.java    |  250 ----
 .../hadoop/HadoopReducerOperatorDescriptor.java |  418 ------
 .../hadoop/HadoopWriteOperatorDescriptor.java   |  241 ----
 .../hadoop/data/AbstractClassBasedDelegate.java |   44 -
 ...HadoopHashTuplePartitionComputerFactory.java |   56 -
 ...artitionerTuplePartitionComputerFactory.java |   62 -
 ...artitionerTuplePartitionComputerFactory.java |   62 -
 .../hadoop/data/KeyBinaryComparatorFactory.java |   42 -
 .../hadoop/data/KeyComparatorFactory.java       |   41 -
 .../data/RawComparingComparatorFactory.java     |   43 -
 ...ritableComparingBinaryComparatorFactory.java |   42 -
 .../dataflow/hadoop/mapreduce/HadoopHelper.java |  272 ----
 .../dataflow/hadoop/mapreduce/HadoopTools.java  |   33 -
 ...hPartitioningShuffleConnectorDescriptor.java |   65 -
 .../hadoop/mapreduce/IInputSplitProvider.java   |   23 -
 .../mapreduce/IInputSplitProviderFactory.java   |   23 -
 .../hadoop/mapreduce/InputFileSplit.java        |  106 --
 .../dataflow/hadoop/mapreduce/KVIterator.java   |  107 --
 .../mapreduce/MapperOperatorDescriptor.java     |  264 ----
 .../hadoop/mapreduce/MarshalledWritable.java    |   65 -
 .../dataflow/hadoop/mapreduce/ReduceWriter.java |  194 ---
 .../mapreduce/ReducerOperatorDescriptor.java    |   92 --
 .../hadoop/mapreduce/ShuffleFrameReader.java    |  215 ---
 .../util/ClasspathBasedHadoopClassFactory.java  |   40 -
 .../dataflow/hadoop/util/DatatypeHelper.java    |  118 --
 .../hadoop/util/DuplicateKeyMapper.java         |   30 -
 .../hadoop/util/IHadoopClassFactory.java        |   28 -
 .../dataflow/hadoop/util/InputSplitsProxy.java  |   82 --
 .../dataflow/hadoop/util/MRContextUtil.java     |   61 -
 .../util/PreappendLongWritableMapper.java       |   32 -
 .../AbstractHadoopOperatorDescriptor.java       |  168 +++
 .../hadoop/HadoopMapperOperatorDescriptor.java  |  446 ++++++
 .../hadoop/HadoopReadOperatorDescriptor.java    |  250 ++++
 .../hadoop/HadoopReducerOperatorDescriptor.java |  418 ++++++
 .../hadoop/HadoopWriteOperatorDescriptor.java   |  241 ++++
 .../hadoop/data/AbstractClassBasedDelegate.java |   44 +
 ...HadoopHashTuplePartitionComputerFactory.java |   56 +
 ...artitionerTuplePartitionComputerFactory.java |   62 +
 ...artitionerTuplePartitionComputerFactory.java |   62 +
 .../hadoop/data/KeyBinaryComparatorFactory.java |   42 +
 .../hadoop/data/KeyComparatorFactory.java       |   41 +
 .../data/RawComparingComparatorFactory.java     |   43 +
 ...ritableComparingBinaryComparatorFactory.java |   42 +
 .../dataflow/hadoop/mapreduce/HadoopHelper.java |  272 ++++
 .../dataflow/hadoop/mapreduce/HadoopTools.java  |   33 +
 ...hPartitioningShuffleConnectorDescriptor.java |   65 +
 .../hadoop/mapreduce/IInputSplitProvider.java   |   23 +
 .../mapreduce/IInputSplitProviderFactory.java   |   23 +
 .../hadoop/mapreduce/InputFileSplit.java        |  106 ++
 .../dataflow/hadoop/mapreduce/KVIterator.java   |  107 ++
 .../mapreduce/MapperOperatorDescriptor.java     |  264 ++++
 .../hadoop/mapreduce/MarshalledWritable.java    |   65 +
 .../dataflow/hadoop/mapreduce/ReduceWriter.java |  194 +++
 .../mapreduce/ReducerOperatorDescriptor.java    |   92 ++
 .../hadoop/mapreduce/ShuffleFrameReader.java    |  215 +++
 .../util/ClasspathBasedHadoopClassFactory.java  |   40 +
 .../dataflow/hadoop/util/DatatypeHelper.java    |  118 ++
 .../hadoop/util/DuplicateKeyMapper.java         |   30 +
 .../hadoop/util/IHadoopClassFactory.java        |   28 +
 .../dataflow/hadoop/util/InputSplitsProxy.java  |   82 ++
 .../dataflow/hadoop/util/MRContextUtil.java     |   61 +
 .../util/PreappendLongWritableMapper.java       |   32 +
 .../dataflow/std/base/AbstractActivityNode.java |   33 -
 .../std/base/AbstractConnectorDescriptor.java   |   66 -
 .../base/AbstractMToNConnectorDescriptor.java   |   46 -
 .../std/base/AbstractOperatorDescriptor.java    |   93 --
 .../std/base/AbstractOperatorNodePushable.java  |   24 -
 ...bstractSingleActivityOperatorDescriptor.java |   47 -
 .../dataflow/std/base/AbstractStateObject.java  |   61 -
 .../AbstractUnaryInputOperatorNodePushable.java |   43 -
 ...tractUnaryInputSinkOperatorNodePushable.java |   25 -
 ...aryInputUnaryOutputOperatorNodePushable.java |   30 -
 ...AbstractUnaryOutputOperatorNodePushable.java |   41 -
 ...ctUnaryOutputSourceOperatorNodePushable.java |   29 -
 .../std/base/IOpenableDataWriterOperator.java   |   21 -
 .../collectors/AbstractPartitionCollector.java  |   49 -
 .../std/collectors/IPartitionAcceptor.java      |   22 -
 .../std/collectors/IPartitionBatchManager.java  |   24 -
 .../std/collectors/InputChannelFrameReader.java |  124 --
 .../NonDeterministicChannelReader.java          |  171 ---
 .../collectors/NonDeterministicFrameReader.java |   65 -
 .../NonDeterministicPartitionBatchManager.java  |   78 --
 .../std/collectors/PartitionCollector.java      |   81 --
 .../std/collectors/SortMergeFrameReader.java    |   82 --
 .../connectors/GlobalHashingLocalityMap.java    |   50 -
 .../std/connectors/HashtableLocalityMap.java    |   72 -
 .../dataflow/std/connectors/ILocalityMap.java   |   27 -
 ...wareMToNPartitioningConnectorDescriptor.java |   87 --
 .../LocalityAwarePartitionDataWriter.java       |  111 --
 .../MToNPartitioningConnectorDescriptor.java    |   60 -
 ...NPartitioningMergingConnectorDescriptor.java |   88 --
 .../MToNReplicatingConnectorDescriptor.java     |   92 --
 .../connectors/OneToOneConnectorDescriptor.java |   90 --
 .../std/connectors/PartitionDataWriter.java     |  102 --
 ...tDeserializedFileScanOperatorDescriptor.java |  114 --
 .../AbstractFileWriteOperatorDescriptor.java    |   96 --
 .../std/file/ConstantFileSplitProvider.java     |   29 -
 .../file/DelimitedDataTupleParserFactory.java   |   95 --
 .../file/FieldCursorForDelimitedDataParser.java |  361 -----
 .../std/file/FileRemoveOperatorDescriptor.java  |   81 --
 .../std/file/FileScanOperatorDescriptor.java    |   74 -
 .../hyracks/dataflow/std/file/FileSplit.java    |   63 -
 .../file/FrameFileWriterOperatorDescriptor.java |   80 --
 .../dataflow/std/file/IFileSplitProvider.java   |   21 -
 .../dataflow/std/file/IRecordReader.java        |   23 -
 .../dataflow/std/file/IRecordWriter.java        |   25 -
 .../hyracks/dataflow/std/file/ITupleParser.java |   24 -
 .../dataflow/std/file/ITupleParserFactory.java  |   24 -
 .../file/LineFileWriteOperatorDescriptor.java   |   60 -
 .../file/PlainFileWriterOperatorDescriptor.java |  133 --
 .../file/RecordFileScanOperatorDescriptor.java  |   82 --
 .../hyracks/dataflow/std/file/RecordWriter.java |   85 --
 ...AccumulatingAggregatorDescriptorFactory.java |   41 -
 .../AbstractRunningAggregatorDescriptor.java    |   41 -
 .../dataflow/std/group/AggregateState.java      |   44 -
 .../DeserializedPreclusteredGroupOperator.java  |  125 --
 .../std/group/FrameToolsForGroupers.java        |   93 --
 .../std/group/HashSpillableTableFactory.java    |  474 -------
 .../std/group/IAggregatorDescriptor.java        |  106 --
 .../std/group/IAggregatorDescriptorFactory.java |   30 -
 .../std/group/IFieldAggregateDescriptor.java    |  120 --
 .../group/IFieldAggregateDescriptorFactory.java |   31 -
 .../dataflow/std/group/IGroupAggregator.java    |   27 -
 .../dataflow/std/group/ISpillableTable.java     |   39 -
 .../std/group/ISpillableTableFactory.java       |   31 -
 .../AvgFieldGroupAggregatorFactory.java         |  172 ---
 .../AvgFieldMergeAggregatorFactory.java         |  171 ---
 .../CountFieldAggregatorFactory.java            |  142 --
 .../FloatSumFieldAggregatorFactory.java         |  157 ---
 .../IntSumFieldAggregatorFactory.java           |  156 ---
 .../MinMaxStringFieldAggregatorFactory.java     |  200 ---
 .../MultiFieldsAggregatorFactory.java           |  169 ---
 .../ExternalGroupBuildOperatorNodePushable.java |  138 --
 .../ExternalGroupMergeOperatorNodePushable.java |  465 -------
 .../ExternalGroupOperatorDescriptor.java        |  139 --
 .../std/group/external/ExternalGroupState.java  |   64 -
 .../std/group/hash/GroupingHashTable.java       |  247 ----
 .../HashGroupBuildOperatorNodePushable.java     |   89 --
 .../group/hash/HashGroupOperatorDescriptor.java |  112 --
 .../HashGroupOutputOperatorNodePushable.java    |   44 -
 .../dataflow/std/group/hash/HashGroupState.java |   51 -
 .../PreclusteredGroupOperatorDescriptor.java    |   51 -
 .../PreclusteredGroupOperatorNodePushable.java  |   72 -
 .../preclustered/PreclusteredGroupWriter.java   |  180 ---
 .../sort/ExternalSortGroupByRunGenerator.java   |   85 --
 .../sort/ExternalSortGroupByRunMerger.java      |  125 --
 .../sort/SortGroupByOperatorDescriptor.java     |  141 --
 .../join/GraceHashJoinOperatorDescriptor.java   |  160 ---
 .../join/GraceHashJoinOperatorNodePushable.java |  154 ---
 ...hJoinPartitionBuildOperatorNodePushable.java |  130 --
 .../std/join/GraceHashJoinPartitionState.java   |   49 -
 .../join/HybridHashJoinOperatorDescriptor.java  |  570 --------
 .../dataflow/std/join/InMemoryHashJoin.java     |  179 ---
 .../InMemoryHashJoinOperatorDescriptor.java     |  254 ----
 .../dataflow/std/join/JoinComparator.java       |   57 -
 .../std/join/JoinComparatorFactory.java         |   40 -
 .../dataflow/std/join/NestedLoopJoin.java       |  219 ---
 .../join/NestedLoopJoinOperatorDescriptor.java  |  217 ---
 .../std/join/OptimizedHybridHashJoin.java       |  672 ---------
 ...timizedHybridHashJoinOperatorDescriptor.java |  706 ----------
 .../DeserializedMapperOperatorDescriptor.java   |   80 --
 .../dataflow/std/map/IDeserializedMapper.java   |   22 -
 .../std/map/IDeserializedMapperFactory.java     |   23 -
 ...eflectionBasedDeserializedMapperFactory.java |   38 -
 .../map/SamplerDeserializedMapperFactory.java   |   43 -
 .../ConstantTupleSourceOperatorDescriptor.java  |   47 -
 ...ConstantTupleSourceOperatorNodePushable.java |   55 -
 .../std/misc/IdentityOperatorDescriptor.java    |   62 -
 .../std/misc/LimitOperatorDescriptor.java       |   94 --
 .../std/misc/MaterializerTaskState.java         |   86 --
 .../misc/MaterializingOperatorDescriptor.java   |  182 ---
 .../std/misc/NullSinkOperatorDescriptor.java    |   55 -
 .../std/misc/PrinterOperatorDescriptor.java     |   69 -
 .../std/misc/SinkOperatorDescriptor.java        |   55 -
 .../std/misc/SplitOperatorDescriptor.java       |  193 ---
 .../std/misc/SplitVectorOperatorDescriptor.java |  189 ---
 .../result/ResultWriterOperatorDescriptor.java  |  121 --
 .../dataflow/std/sort/AbstractFrameSorter.java  |  186 ---
 .../std/sort/AbstractSortRunGenerator.java      |   77 --
 .../sort/AbstractSorterOperatorDescriptor.java  |  197 ---
 .../hyracks/dataflow/std/sort/Algorithm.java    |   21 -
 .../sort/ExternalSortOperatorDescriptor.java    |   98 --
 .../std/sort/ExternalSortRunGenerator.java      |  112 --
 .../std/sort/ExternalSortRunMerger.java         |  271 ----
 .../dataflow/std/sort/FrameSorterMergeSort.java |  159 ---
 .../dataflow/std/sort/FrameSorterQuickSort.java |  149 --
 .../dataflow/std/sort/HeapSortRunGenerator.java |   99 --
 .../std/sort/HybridTopKSortRunGenerator.java    |  109 --
 .../hyracks/dataflow/std/sort/IFrameSorter.java |   28 -
 .../dataflow/std/sort/IRunGenerator.java        |   31 -
 .../ics/hyracks/dataflow/std/sort/ISorter.java  |   33 -
 .../hyracks/dataflow/std/sort/ITupleSorter.java |   26 -
 .../sort/InMemorySortOperatorDescriptor.java    |  180 ---
 .../std/sort/RunAndMaxFrameSizePair.java        |   32 -
 .../std/sort/RunMergingFrameReader.java         |  195 ---
 .../std/sort/TopKSorterOperatorDescriptor.java  |   62 -
 .../dataflow/std/sort/TupleSorterHeapSort.java  |  269 ----
 .../sort/buffermanager/EnumFreeSlotPolicy.java  |   22 -
 .../FrameFreeSlotBiggestFirst.java              |   97 --
 .../buffermanager/FrameFreeSlotLastFit.java     |   81 --
 .../buffermanager/FrameFreeSlotSmallestFit.java |   59 -
 .../sort/buffermanager/IFrameBufferManager.java |   68 -
 .../buffermanager/IFrameFreeSlotPolicy.java     |   44 -
 .../std/sort/buffermanager/IFramePool.java      |   48 -
 .../buffermanager/ITupleBufferAccessor.java     |   36 -
 .../sort/buffermanager/ITupleBufferManager.java |   42 -
 .../VariableFrameMemoryManager.java             |  132 --
 .../sort/buffermanager/VariableFramePool.java   |  200 ---
 .../VariableTupleMemoryManager.java             |  203 ---
 .../sort/util/DeletableFrameTupleAppender.java  |  244 ----
 .../std/sort/util/GroupFrameAccessor.java       |  170 ---
 .../dataflow/std/sort/util/GroupVSizeFrame.java |   46 -
 .../IAppendDeletableFrameTupleAccessor.java     |   72 -
 .../dataflow/std/structures/AbstractHeap.java   |  156 ---
 .../hyracks/dataflow/std/structures/IHeap.java  |   44 -
 .../dataflow/std/structures/IMaxHeap.java       |   43 -
 .../dataflow/std/structures/IMinHeap.java       |   42 -
 .../dataflow/std/structures/IMinMaxHeap.java    |   18 -
 .../dataflow/std/structures/IResetable.java     |   20 -
 .../std/structures/IResetableComparable.java    |   19 -
 .../structures/IResetableComparableFactory.java |   20 -
 .../std/structures/ISerializableTable.java      |   32 -
 .../dataflow/std/structures/MaxHeap.java        |   63 -
 .../dataflow/std/structures/MinHeap.java        |   62 -
 .../dataflow/std/structures/MinMaxHeap.java     |  217 ---
 .../std/structures/SerializableHashTable.java   |  285 ----
 .../dataflow/std/structures/TuplePointer.java   |   61 -
 .../std/union/UnionAllOperatorDescriptor.java   |  123 --
 .../util/DeserializedOperatorNodePushable.java  |   75 -
 .../ics/hyracks/dataflow/std/util/MathUtil.java |   50 -
 .../dataflow/std/util/ReferenceEntry.java       |   83 --
 .../std/util/ReferencedPriorityQueue.java       |  139 --
 .../dataflow/std/util/SelectionTree.java        |  107 --
 .../std/util/StringSerializationUtils.java      |   27 -
 .../dataflow/std/base/AbstractActivityNode.java |   33 +
 .../std/base/AbstractConnectorDescriptor.java   |   66 +
 .../base/AbstractMToNConnectorDescriptor.java   |   46 +
 .../std/base/AbstractOperatorDescriptor.java    |   93 ++
 .../std/base/AbstractOperatorNodePushable.java  |   24 +
 ...bstractSingleActivityOperatorDescriptor.java |   47 +
 .../dataflow/std/base/AbstractStateObject.java  |   61 +
 .../AbstractUnaryInputOperatorNodePushable.java |   43 +
 ...tractUnaryInputSinkOperatorNodePushable.java |   25 +
 ...aryInputUnaryOutputOperatorNodePushable.java |   30 +
 ...AbstractUnaryOutputOperatorNodePushable.java |   41 +
 ...ctUnaryOutputSourceOperatorNodePushable.java |   29 +
 .../std/base/IOpenableDataWriterOperator.java   |   21 +
 .../collectors/AbstractPartitionCollector.java  |   49 +
 .../std/collectors/IPartitionAcceptor.java      |   22 +
 .../std/collectors/IPartitionBatchManager.java  |   24 +
 .../std/collectors/InputChannelFrameReader.java |  124 ++
 .../NonDeterministicChannelReader.java          |  171 +++
 .../collectors/NonDeterministicFrameReader.java |   65 +
 .../NonDeterministicPartitionBatchManager.java  |   78 ++
 .../std/collectors/PartitionCollector.java      |   81 ++
 .../std/collectors/SortMergeFrameReader.java    |   82 ++
 .../connectors/GlobalHashingLocalityMap.java    |   50 +
 .../std/connectors/HashtableLocalityMap.java    |   72 +
 .../dataflow/std/connectors/ILocalityMap.java   |   27 +
 ...wareMToNPartitioningConnectorDescriptor.java |   87 ++
 .../LocalityAwarePartitionDataWriter.java       |  111 ++
 .../MToNPartitioningConnectorDescriptor.java    |   60 +
 ...NPartitioningMergingConnectorDescriptor.java |   88 ++
 .../MToNReplicatingConnectorDescriptor.java     |   92 ++
 .../connectors/OneToOneConnectorDescriptor.java |   90 ++
 .../std/connectors/PartitionDataWriter.java     |  102 ++
 ...tDeserializedFileScanOperatorDescriptor.java |  114 ++
 .../AbstractFileWriteOperatorDescriptor.java    |   96 ++
 .../std/file/ConstantFileSplitProvider.java     |   29 +
 .../file/DelimitedDataTupleParserFactory.java   |   95 ++
 .../file/FieldCursorForDelimitedDataParser.java |  361 +++++
 .../std/file/FileRemoveOperatorDescriptor.java  |   81 ++
 .../std/file/FileScanOperatorDescriptor.java    |   74 +
 .../hyracks/dataflow/std/file/FileSplit.java    |   63 +
 .../file/FrameFileWriterOperatorDescriptor.java |   80 ++
 .../dataflow/std/file/IFileSplitProvider.java   |   21 +
 .../dataflow/std/file/IRecordReader.java        |   23 +
 .../dataflow/std/file/IRecordWriter.java        |   25 +
 .../hyracks/dataflow/std/file/ITupleParser.java |   24 +
 .../dataflow/std/file/ITupleParserFactory.java  |   24 +
 .../file/LineFileWriteOperatorDescriptor.java   |   60 +
 .../file/PlainFileWriterOperatorDescriptor.java |  133 ++
 .../file/RecordFileScanOperatorDescriptor.java  |   82 ++
 .../hyracks/dataflow/std/file/RecordWriter.java |   85 ++
 ...AccumulatingAggregatorDescriptorFactory.java |   41 +
 .../AbstractRunningAggregatorDescriptor.java    |   41 +
 .../dataflow/std/group/AggregateState.java      |   44 +
 .../DeserializedPreclusteredGroupOperator.java  |  125 ++
 .../std/group/FrameToolsForGroupers.java        |   93 ++
 .../std/group/HashSpillableTableFactory.java    |  474 +++++++
 .../std/group/IAggregatorDescriptor.java        |  106 ++
 .../std/group/IAggregatorDescriptorFactory.java |   30 +
 .../std/group/IFieldAggregateDescriptor.java    |  120 ++
 .../group/IFieldAggregateDescriptorFactory.java |   31 +
 .../dataflow/std/group/IGroupAggregator.java    |   27 +
 .../dataflow/std/group/ISpillableTable.java     |   39 +
 .../std/group/ISpillableTableFactory.java       |   31 +
 .../AvgFieldGroupAggregatorFactory.java         |  172 +++
 .../AvgFieldMergeAggregatorFactory.java         |  171 +++
 .../CountFieldAggregatorFactory.java            |  142 ++
 .../FloatSumFieldAggregatorFactory.java         |  157 +++
 .../IntSumFieldAggregatorFactory.java           |  156 +++
 .../MinMaxStringFieldAggregatorFactory.java     |  200 +++
 .../MultiFieldsAggregatorFactory.java           |  169 +++
 .../ExternalGroupBuildOperatorNodePushable.java |  138 ++
 .../ExternalGroupMergeOperatorNodePushable.java |  465 +++++++
 .../ExternalGroupOperatorDescriptor.java        |  139 ++
 .../std/group/external/ExternalGroupState.java  |   64 +
 .../std/group/hash/GroupingHashTable.java       |  247 ++++
 .../HashGroupBuildOperatorNodePushable.java     |   89 ++
 .../group/hash/HashGroupOperatorDescriptor.java |  112 ++
 .../HashGroupOutputOperatorNodePushable.java    |   44 +
 .../dataflow/std/group/hash/HashGroupState.java |   51 +
 .../PreclusteredGroupOperatorDescriptor.java    |   51 +
 .../PreclusteredGroupOperatorNodePushable.java  |   72 +
 .../preclustered/PreclusteredGroupWriter.java   |  180 +++
 .../sort/ExternalSortGroupByRunGenerator.java   |   85 ++
 .../sort/ExternalSortGroupByRunMerger.java      |  125 ++
 .../sort/SortGroupByOperatorDescriptor.java     |  141 ++
 .../join/GraceHashJoinOperatorDescriptor.java   |  160 +++
 .../join/GraceHashJoinOperatorNodePushable.java |  154 +++
 ...hJoinPartitionBuildOperatorNodePushable.java |  130 ++
 .../std/join/GraceHashJoinPartitionState.java   |   49 +
 .../join/HybridHashJoinOperatorDescriptor.java  |  570 ++++++++
 .../dataflow/std/join/InMemoryHashJoin.java     |  179 +++
 .../InMemoryHashJoinOperatorDescriptor.java     |  254 ++++
 .../dataflow/std/join/JoinComparator.java       |   57 +
 .../std/join/JoinComparatorFactory.java         |   40 +
 .../dataflow/std/join/NestedLoopJoin.java       |  219 +++
 .../join/NestedLoopJoinOperatorDescriptor.java  |  217 +++
 .../std/join/OptimizedHybridHashJoin.java       |  672 +++++++++
 ...timizedHybridHashJoinOperatorDescriptor.java |  706 ++++++++++
 .../DeserializedMapperOperatorDescriptor.java   |   80 ++
 .../dataflow/std/map/IDeserializedMapper.java   |   22 +
 .../std/map/IDeserializedMapperFactory.java     |   23 +
 ...eflectionBasedDeserializedMapperFactory.java |   38 +
 .../map/SamplerDeserializedMapperFactory.java   |   43 +
 .../ConstantTupleSourceOperatorDescriptor.java  |   47 +
 ...ConstantTupleSourceOperatorNodePushable.java |   55 +
 .../std/misc/IdentityOperatorDescriptor.java    |   62 +
 .../std/misc/LimitOperatorDescriptor.java       |   94 ++
 .../std/misc/MaterializerTaskState.java         |   86 ++
 .../misc/MaterializingOperatorDescriptor.java   |  182 +++
 .../std/misc/NullSinkOperatorDescriptor.java    |   55 +
 .../std/misc/PrinterOperatorDescriptor.java     |   69 +
 .../std/misc/SinkOperatorDescriptor.java        |   55 +
 .../std/misc/SplitOperatorDescriptor.java       |  193 +++
 .../std/misc/SplitVectorOperatorDescriptor.java |  189 +++
 .../result/ResultWriterOperatorDescriptor.java  |  121 ++
 .../dataflow/std/sort/AbstractFrameSorter.java  |  186 +++
 .../std/sort/AbstractSortRunGenerator.java      |   77 ++
 .../sort/AbstractSorterOperatorDescriptor.java  |  197 +++
 .../hyracks/dataflow/std/sort/Algorithm.java    |   21 +
 .../sort/ExternalSortOperatorDescriptor.java    |   98 ++
 .../std/sort/ExternalSortRunGenerator.java      |  112 ++
 .../std/sort/ExternalSortRunMerger.java         |  271 ++++
 .../dataflow/std/sort/FrameSorterMergeSort.java |  159 +++
 .../dataflow/std/sort/FrameSorterQuickSort.java |  149 ++
 .../dataflow/std/sort/HeapSortRunGenerator.java |   99 ++
 .../std/sort/HybridTopKSortRunGenerator.java    |  109 ++
 .../hyracks/dataflow/std/sort/IFrameSorter.java |   28 +
 .../dataflow/std/sort/IRunGenerator.java        |   31 +
 .../hyracks/dataflow/std/sort/ISorter.java      |   33 +
 .../hyracks/dataflow/std/sort/ITupleSorter.java |   26 +
 .../sort/InMemorySortOperatorDescriptor.java    |  180 +++
 .../std/sort/RunAndMaxFrameSizePair.java        |   32 +
 .../std/sort/RunMergingFrameReader.java         |  195 +++
 .../std/sort/TopKSorterOperatorDescriptor.java  |   62 +
 .../dataflow/std/sort/TupleSorterHeapSort.java  |  269 ++++
 .../sort/buffermanager/EnumFreeSlotPolicy.java  |   22 +
 .../FrameFreeSlotBiggestFirst.java              |   97 ++
 .../buffermanager/FrameFreeSlotLastFit.java     |   81 ++
 .../buffermanager/FrameFreeSlotSmallestFit.java |   59 +
 .../sort/buffermanager/IFrameBufferManager.java |   68 +
 .../buffermanager/IFrameFreeSlotPolicy.java     |   44 +
 .../std/sort/buffermanager/IFramePool.java      |   48 +
 .../buffermanager/ITupleBufferAccessor.java     |   36 +
 .../sort/buffermanager/ITupleBufferManager.java |   42 +
 .../VariableFrameMemoryManager.java             |  132 ++
 .../sort/buffermanager/VariableFramePool.java   |  200 +++
 .../VariableTupleMemoryManager.java             |  203 +++
 .../sort/util/DeletableFrameTupleAppender.java  |  244 ++++
 .../std/sort/util/GroupFrameAccessor.java       |  170 +++
 .../dataflow/std/sort/util/GroupVSizeFrame.java |   46 +
 .../IAppendDeletableFrameTupleAccessor.java     |   72 +
 .../dataflow/std/structures/AbstractHeap.java   |  156 +++
 .../hyracks/dataflow/std/structures/IHeap.java  |   44 +
 .../dataflow/std/structures/IMaxHeap.java       |   43 +
 .../dataflow/std/structures/IMinHeap.java       |   42 +
 .../dataflow/std/structures/IMinMaxHeap.java    |   18 +
 .../dataflow/std/structures/IResetable.java     |   20 +
 .../std/structures/IResetableComparable.java    |   19 +
 .../structures/IResetableComparableFactory.java |   20 +
 .../std/structures/ISerializableTable.java      |   32 +
 .../dataflow/std/structures/MaxHeap.java        |   63 +
 .../dataflow/std/structures/MinHeap.java        |   62 +
 .../dataflow/std/structures/MinMaxHeap.java     |  217 +++
 .../std/structures/SerializableHashTable.java   |  285 ++++
 .../dataflow/std/structures/TuplePointer.java   |   61 +
 .../std/union/UnionAllOperatorDescriptor.java   |  123 ++
 .../util/DeserializedOperatorNodePushable.java  |   75 +
 .../hyracks/dataflow/std/util/MathUtil.java     |   50 +
 .../dataflow/std/util/ReferenceEntry.java       |   83 ++
 .../std/util/ReferencedPriorityQueue.java       |  139 ++
 .../dataflow/std/util/SelectionTree.java        |  107 ++
 .../std/util/StringSerializationUtils.java      |   27 +
 .../ics/hyracks/dataflow/std/sort/Utility.java  |   23 -
 .../dataflow/std/sort/buffermanager/Common.java |   26 -
 .../FrameFreeSlotBestFitUsingTreeMapTest.java   |   60 -
 .../FrameFreeSlotBiggestFirstTest.java          |   70 -
 .../buffermanager/FrameFreeSlotLastFitTest.java |   86 --
 .../buffermanager/VariableFramePoolTest.java    |  216 ---
 .../VariableFramesMemoryManagerTest.java        |  170 ---
 .../VariableTupleMemoryManagerTest.java         |  230 ---
 .../util/DeletableFrameTupleAppenderTest.java   |  233 ----
 .../std/structures/AbstracHeapTest.java         |   86 --
 .../dataflow/std/structures/MaxHeapTest.java    |   99 --
 .../dataflow/std/structures/MinHeapTest.java    |  102 --
 .../dataflow/std/structures/MinMaxHeapTest.java |  109 --
 .../std/test/util/SelectionTreeTest.java        |   69 -
 .../ics/hyracks/dataflow/std/util/MathTest.java |   40 -
 .../hyracks/dataflow/std/sort/Utility.java      |   23 +
 .../dataflow/std/sort/buffermanager/Common.java |   26 +
 .../FrameFreeSlotBestFitUsingTreeMapTest.java   |   60 +
 .../FrameFreeSlotBiggestFirstTest.java          |   70 +
 .../buffermanager/FrameFreeSlotLastFitTest.java |   86 ++
 .../buffermanager/VariableFramePoolTest.java    |  216 +++
 .../VariableFramesMemoryManagerTest.java        |  170 +++
 .../VariableTupleMemoryManagerTest.java         |  230 +++
 .../util/DeletableFrameTupleAppenderTest.java   |  233 ++++
 .../std/structures/AbstracHeapTest.java         |   86 ++
 .../dataflow/std/structures/MaxHeapTest.java    |   99 ++
 .../dataflow/std/structures/MinHeapTest.java    |  102 ++
 .../dataflow/std/structures/MinMaxHeapTest.java |  109 ++
 .../std/test/util/SelectionTreeTest.java        |   69 +
 .../hyracks/dataflow/std/util/MathTest.java     |   40 +
 .../btree/client/InsertPipelineExample.java     |  205 ---
 .../examples/btree/client/JobHelper.java        |   42 -
 .../client/PrimaryIndexBulkLoadExample.java     |  175 ---
 .../btree/client/PrimaryIndexSearchExample.java |  162 ---
 .../client/SecondaryIndexBulkLoadExample.java   |  161 ---
 .../client/SecondaryIndexSearchExample.java     |  207 ---
 .../btree/client/InsertPipelineExample.java     |  205 +++
 .../examples/btree/client/JobHelper.java        |   42 +
 .../client/PrimaryIndexBulkLoadExample.java     |  175 +++
 .../btree/client/PrimaryIndexSearchExample.java |  162 +++
 .../client/SecondaryIndexBulkLoadExample.java   |  161 +++
 .../client/SecondaryIndexSearchExample.java     |  207 +++
 .../btree/helper/DataGenOperatorDescriptor.java |  162 ---
 .../helper/IndexLifecycleManagerProvider.java   |   29 -
 .../btree/helper/NCApplicationEntryPoint.java   |   36 -
 .../examples/btree/helper/RuntimeContext.java   |   92 --
 .../btree/helper/StorageManagerInterface.java   |   52 -
 .../btree/helper/DataGenOperatorDescriptor.java |  162 +++
 .../helper/IndexLifecycleManagerProvider.java   |   29 +
 .../btree/helper/NCApplicationEntryPoint.java   |   36 +
 .../examples/btree/helper/RuntimeContext.java   |   92 ++
 .../btree/helper/StorageManagerInterface.java   |   52 +
 .../compat/client/WordCountCompatibility.java   |   23 -
 .../compat/client/WordCountCompatibility.java   |   23 +
 .../hyracks/examples/wordcount/WordCount.java   |   62 -
 .../hyracks/examples/wordcount/WordCount.java   |   62 +
 .../compat/test/WordCountCompatibilityIT.java   |   37 -
 .../compat/test/WordCountCompatibilityIT.java   |   37 +
 .../am/btree/AbstractBTreeOperatorTest.java     |  334 -----
 .../tests/am/btree/BTreeOperatorTestHelper.java |   28 -
 .../BTreePrimaryIndexScanOperatorTest.java      |   99 --
 .../BTreePrimaryIndexSearchOperatorTest.java    |  104 --
 .../BTreePrimaryIndexStatsOperatorTest.java     |   69 -
 .../BTreeSecondaryIndexInsertOperatorTest.java  |  124 --
 .../BTreeSecondaryIndexSearchOperatorTest.java  |  124 --
 .../BTreeSecondaryIndexUpsertOperatorTest.java  |  123 --
 .../am/common/ITreeIndexOperatorTestHelper.java |   24 -
 .../am/common/LSMTreeOperatorTestHelper.java    |   76 -
 .../tests/am/common/TreeOperatorTestHelper.java |   46 -
 .../AbstractfWordInvertedIndexTest.java         |  363 -----
 .../BinaryTokenizerOperatorTest.java            |  103 --
 .../PartitionedWordInvertedIndexTest.java       |   65 -
 .../am/invertedindex/WordInvertedIndexTest.java |   60 -
 .../lsm/btree/LSMBTreeOperatorTestHelper.java   |   49 -
 .../LSMBTreePrimaryIndexScanOperatorTest.java   |   34 -
 .../LSMBTreePrimaryIndexSearchOperatorTest.java |   33 -
 ...SMBTreeSecondaryIndexInsertOperatorTest.java |   33 -
 ...SMBTreeSecondaryIndexSearchOperatorTest.java |   33 -
 .../lsm/rtree/LSMRTreeOperatorTestHelper.java   |   56 -
 ...SMRTreeSecondaryIndexInsertOperatorTest.java |   46 -
 ...SMRTreeSecondaryIndexSearchOperatorTest.java |   45 -
 ...eWithAntiMatterTuplesOperatorTestHelper.java |   55 -
 ...rTuplesSecondaryIndexInsertOperatorTest.java |   45 -
 ...rTuplesSecondaryIndexSearchOperatorTest.java |   45 -
 .../am/rtree/AbstractRTreeOperatorTest.java     |  397 ------
 .../tests/am/rtree/RTreeOperatorTestHelper.java |   33 -
 .../RTreeSecondaryIndexInsertOperatorTest.java  |  139 --
 .../RTreeSecondaryIndexScanOperatorTest.java    |  121 --
 .../RTreeSecondaryIndexSearchOperatorTest.java  |  135 --
 .../RTreeSecondaryIndexStatsOperatorTest.java   |   84 --
 .../comm/SerializationDeserializationTest.java  |  155 ---
 .../integration/AbstractIntegrationTest.java    |  228 ---
 .../AbstractMultiNCIntegrationTest.java         |  195 ---
 .../tests/integration/AggregationTest.java      |  913 ------------
 .../tests/integration/CountOfCountsTest.java    |  275 ----
 .../integration/LocalityAwareConnectorTest.java |  215 ---
 .../integration/OptimizedSortMergeTest.java     |  167 ---
 .../tests/integration/ScanPrintTest.java        |  159 ---
 .../tests/integration/SortMergeTest.java        |  148 --
 .../tests/integration/SplitOperatorTest.java    |  112 --
 ...TPCHCustomerOptimizedHybridHashJoinTest.java |  285 ----
 .../TPCHCustomerOrderHashJoinTest.java          | 1028 --------------
 .../TPCHCustomerOrderNestedLoopJoinTest.java    |  434 ------
 .../hyracks/tests/integration/UnionTest.java    |   84 --
 .../integration/VSizeFrameSortMergeTest.java    |  118 --
 .../tests/unit/AbstractRunGeneratorTest.java    |  279 ----
 .../unit/ExternalSortRunGeneratorTest.java      |   32 -
 .../tests/unit/HeapSortRunGeneratorTest.java    |   37 -
 .../tests/unit/HybridSortRunGenerator.java      |   30 -
 .../tests/unit/RunMergingFrameReaderTest.java   |  409 ------
 .../tests/unit/TopKRunGeneratorTest.java        |  208 ---
 .../tests/util/NoopNullWriterFactory.java       |   45 -
 .../util/ResultSerializerFactoryProvider.java   |   76 -
 .../am/btree/AbstractBTreeOperatorTest.java     |  334 +++++
 .../tests/am/btree/BTreeOperatorTestHelper.java |   28 +
 .../BTreePrimaryIndexScanOperatorTest.java      |   99 ++
 .../BTreePrimaryIndexSearchOperatorTest.java    |  104 ++
 .../BTreePrimaryIndexStatsOperatorTest.java     |   69 +
 .../BTreeSecondaryIndexInsertOperatorTest.java  |  124 ++
 .../BTreeSecondaryIndexSearchOperatorTest.java  |  124 ++
 .../BTreeSecondaryIndexUpsertOperatorTest.java  |  123 ++
 .../am/common/ITreeIndexOperatorTestHelper.java |   24 +
 .../am/common/LSMTreeOperatorTestHelper.java    |   76 +
 .../tests/am/common/TreeOperatorTestHelper.java |   46 +
 .../lsm/btree/LSMBTreeOperatorTestHelper.java   |   49 +
 .../LSMBTreePrimaryIndexScanOperatorTest.java   |   34 +
 .../LSMBTreePrimaryIndexSearchOperatorTest.java |   33 +
 ...SMBTreeSecondaryIndexInsertOperatorTest.java |   33 +
 ...SMBTreeSecondaryIndexSearchOperatorTest.java |   33 +
 .../lsm/rtree/LSMRTreeOperatorTestHelper.java   |   56 +
 ...SMRTreeSecondaryIndexInsertOperatorTest.java |   46 +
 ...SMRTreeSecondaryIndexSearchOperatorTest.java |   45 +
 ...eWithAntiMatterTuplesOperatorTestHelper.java |   55 +
 ...rTuplesSecondaryIndexInsertOperatorTest.java |   45 +
 ...rTuplesSecondaryIndexSearchOperatorTest.java |   45 +
 .../am/rtree/AbstractRTreeOperatorTest.java     |  397 ++++++
 .../tests/am/rtree/RTreeOperatorTestHelper.java |   33 +
 .../RTreeSecondaryIndexInsertOperatorTest.java  |  139 ++
 .../RTreeSecondaryIndexScanOperatorTest.java    |  121 ++
 .../RTreeSecondaryIndexSearchOperatorTest.java  |  135 ++
 .../RTreeSecondaryIndexStatsOperatorTest.java   |   84 ++
 .../comm/SerializationDeserializationTest.java  |  155 +++
 .../integration/AbstractIntegrationTest.java    |  228 +++
 .../AbstractMultiNCIntegrationTest.java         |  195 +++
 .../tests/integration/AggregationTest.java      |  913 ++++++++++++
 .../tests/integration/CountOfCountsTest.java    |  275 ++++
 .../integration/LocalityAwareConnectorTest.java |  215 +++
 .../integration/OptimizedSortMergeTest.java     |  167 +++
 .../tests/integration/ScanPrintTest.java        |  159 +++
 .../tests/integration/SortMergeTest.java        |  148 ++
 .../tests/integration/SplitOperatorTest.java    |  112 ++
 ...TPCHCustomerOptimizedHybridHashJoinTest.java |  285 ++++
 .../TPCHCustomerOrderHashJoinTest.java          | 1028 ++++++++++++++
 .../TPCHCustomerOrderNestedLoopJoinTest.java    |  434 ++++++
 .../hyracks/tests/integration/UnionTest.java    |   84 ++
 .../integration/VSizeFrameSortMergeTest.java    |  118 ++
 .../tests/unit/AbstractRunGeneratorTest.java    |  279 ++++
 .../unit/ExternalSortRunGeneratorTest.java      |   32 +
 .../tests/unit/HeapSortRunGeneratorTest.java    |   37 +
 .../tests/unit/HybridSortRunGenerator.java      |   30 +
 .../tests/unit/RunMergingFrameReaderTest.java   |  409 ++++++
 .../tests/unit/TopKRunGeneratorTest.java        |  208 +++
 .../tests/util/NoopNullWriterFactory.java       |   45 +
 .../util/ResultSerializerFactoryProvider.java   |   76 +
 .../shutdown/test/ClusterShutdownIT.java        |   56 -
 .../shutdown/test/ClusterShutdownIT.java        |   56 +
 .../text/client/ExternalGroupClient.java        |  321 -----
 .../examples/text/client/WordCountMain.java     |  205 ---
 .../text/client/ExternalGroupClient.java        |  321 +++++
 .../examples/text/client/WordCountMain.java     |  205 +++
 .../examples/text/WordTupleParserFactory.java   |  175 ---
 .../examples/text/WordTupleParserFactory.java   |  175 +++
 .../hyracks/examples/text/test/WordCountIT.java |   39 -
 .../hyracks/examples/text/test/WordCountIT.java |   39 +
 .../hyracks/examples/tpch/client/Common.java    |   83 --
 .../ics/hyracks/examples/tpch/client/Join.java  |  320 -----
 .../ics/hyracks/examples/tpch/client/Sort.java  |  165 ---
 .../hyracks/examples/tpch/client/Common.java    |   83 ++
 .../hyracks/examples/tpch/client/Join.java      |  320 +++++
 .../hyracks/examples/tpch/client/Sort.java      |  165 +++
 .../hadoop/compat/client/HyracksClient.java     |   83 --
 .../hadoop/compat/client/HyracksRunningJob.java |  204 ---
 .../compat/driver/CompatibilityLayer.java       |  204 ---
 .../hadoop/compat/util/CompatibilityConfig.java |   37 -
 .../compat/util/ConfigurationConstants.java     |   25 -
 .../hadoop/compat/util/DCacheHandler.java       |  106 --
 .../hadoop/compat/util/HadoopAdapter.java       |  423 ------
 .../hyracks/hadoop/compat/util/Utilities.java   |  133 --
 .../hadoop/compat/client/HyracksClient.java     |   83 ++
 .../hadoop/compat/client/HyracksRunningJob.java |  204 +++
 .../compat/driver/CompatibilityLayer.java       |  204 +++
 .../hadoop/compat/util/CompatibilityConfig.java |   37 +
 .../compat/util/ConfigurationConstants.java     |   25 +
 .../hadoop/compat/util/DCacheHandler.java       |  106 ++
 .../hadoop/compat/util/HadoopAdapter.java       |  423 ++++++
 .../hyracks/hadoop/compat/util/Utilities.java   |  133 ++
 .../uci/ics/hyracks/hdfs/ContextFactory.java    |   53 -
 .../ics/hyracks/hdfs/MiniDFSClusterFactory.java |   31 -
 .../org/apache/hyracks/hdfs/ContextFactory.java |   53 +
 .../hyracks/hdfs/MiniDFSClusterFactory.java     |   31 +
 .../uci/ics/hyracks/hdfs/ContextFactory.java    |   54 -
 .../ics/hyracks/hdfs/MiniDFSClusterFactory.java |   34 -
 .../org/apache/hyracks/hdfs/ContextFactory.java |   54 +
 .../hyracks/hdfs/MiniDFSClusterFactory.java     |   34 +
 .../ics/hyracks/hdfs/api/IKeyValueParser.java   |   58 -
 .../hdfs/api/IKeyValueParserFactory.java        |   42 -
 .../uci/ics/hyracks/hdfs/api/INcCollection.java |   25 -
 .../hyracks/hdfs/api/INcCollectionBuilder.java  |   32 -
 .../uci/ics/hyracks/hdfs/api/ITupleWriter.java  |   57 -
 .../hyracks/hdfs/api/ITupleWriterFactory.java   |   35 -
 .../ics/hyracks/hdfs/dataflow/ConfFactory.java  |   55 -
 .../dataflow/HDFSReadOperatorDescriptor.java    |  146 --
 .../dataflow/HDFSWriteOperatorDescriptor.java   |  131 --
 .../hdfs/dataflow/InputSplitsFactory.java       |  106 --
 .../hdfs/lib/RawBinaryComparatorFactory.java    |   46 -
 .../hdfs/lib/RawBinaryHashFunctionFactory.java  |   43 -
 .../hdfs/lib/TextKeyValueParserFactory.java     |   65 -
 .../hdfs/lib/TextTupleWriterFactory.java        |   60 -
 .../IPProximityNcCollectionBuilder.java         |  144 --
 .../scheduler/RackAwareNcCollectionBuilder.java |  225 ---
 .../ics/hyracks/hdfs/scheduler/Scheduler.java   |  404 ------
 .../ics/hyracks/hdfs2/dataflow/ConfFactory.java |   54 -
 .../hdfs2/dataflow/FileSplitsFactory.java       |  106 --
 .../dataflow/HDFSReadOperatorDescriptor.java    |  163 ---
 .../dataflow/HDFSWriteOperatorDescriptor.java   |  130 --
 .../ics/hyracks/hdfs2/scheduler/Scheduler.java  |   99 --
 .../hdfs2/scheduler/WrappedFileSplit.java       |   65 -
 .../hyracks/hdfs/api/IKeyValueParser.java       |   58 +
 .../hdfs/api/IKeyValueParserFactory.java        |   42 +
 .../apache/hyracks/hdfs/api/INcCollection.java  |   25 +
 .../hyracks/hdfs/api/INcCollectionBuilder.java  |   32 +
 .../apache/hyracks/hdfs/api/ITupleWriter.java   |   57 +
 .../hyracks/hdfs/api/ITupleWriterFactory.java   |   35 +
 .../hyracks/hdfs/dataflow/ConfFactory.java      |   55 +
 .../dataflow/HDFSReadOperatorDescriptor.java    |  146 ++
 .../dataflow/HDFSWriteOperatorDescriptor.java   |  131 ++
 .../hdfs/dataflow/InputSplitsFactory.java       |  106 ++
 .../hdfs/lib/RawBinaryComparatorFactory.java    |   46 +
 .../hdfs/lib/RawBinaryHashFunctionFactory.java  |   43 +
 .../hdfs/lib/TextKeyValueParserFactory.java     |   65 +
 .../hdfs/lib/TextTupleWriterFactory.java        |   60 +
 .../IPProximityNcCollectionBuilder.java         |  144 ++
 .../scheduler/RackAwareNcCollectionBuilder.java |  225 +++
 .../hyracks/hdfs/scheduler/Scheduler.java       |  404 ++++++
 .../hyracks/hdfs2/dataflow/ConfFactory.java     |   54 +
 .../hdfs2/dataflow/FileSplitsFactory.java       |  106 ++
 .../dataflow/HDFSReadOperatorDescriptor.java    |  163 +++
 .../dataflow/HDFSWriteOperatorDescriptor.java   |  130 ++
 .../hyracks/hdfs2/scheduler/Scheduler.java      |   99 ++
 .../hdfs2/scheduler/WrappedFileSplit.java       |   65 +
 .../ics/hyracks/hdfs/dataflow/DataflowTest.java |  201 ---
 .../hyracks/hdfs/scheduler/SchedulerTest.java   |  281 ----
 .../ics/hyracks/hdfs/utils/HyracksUtils.java    |   98 --
 .../uci/ics/hyracks/hdfs/utils/TestUtils.java   |   93 --
 .../hyracks/hdfs2/dataflow/DataflowTest.java    |  208 ---
 .../hyracks/hdfs2/scheduler/SchedulerTest.java  |  190 ---
 .../hyracks/hdfs/dataflow/DataflowTest.java     |  201 +++
 .../hyracks/hdfs/scheduler/SchedulerTest.java   |  281 ++++
 .../apache/hyracks/hdfs/utils/HyracksUtils.java |   98 ++
 .../apache/hyracks/hdfs/utils/TestUtils.java    |   93 ++
 .../hyracks/hdfs2/dataflow/DataflowTest.java    |  208 +++
 .../hyracks/hdfs2/scheduler/SchedulerTest.java  |  190 +++
 .../edu/uci/ics/hyracks/ipc/api/IIPCHandle.java |   31 -
 .../java/edu/uci/ics/hyracks/ipc/api/IIPCI.java |   19 -
 .../hyracks/ipc/api/IPCPerformanceCounters.java |   66 -
 .../ipc/api/IPayloadSerializerDeserializer.java |   27 -
 .../ics/hyracks/ipc/api/IResponseCallback.java  |   19 -
 .../uci/ics/hyracks/ipc/api/RPCInterface.java   |   98 --
 .../hyracks/ipc/exceptions/IPCException.java    |   35 -
 .../uci/ics/hyracks/ipc/impl/HandleState.java   |   24 -
 .../hyracks/ipc/impl/IPCConnectionManager.java  |  343 -----
 .../edu/uci/ics/hyracks/ipc/impl/IPCHandle.java |  191 ---
 .../edu/uci/ics/hyracks/ipc/impl/IPCSystem.java |  101 --
 ...ationBasedPayloadSerializerDeserializer.java |   67 -
 .../edu/uci/ics/hyracks/ipc/impl/Message.java   |  127 --
 .../org/apache/hyracks/ipc/api/IIPCHandle.java  |   31 +
 .../java/org/apache/hyracks/ipc/api/IIPCI.java  |   19 +
 .../hyracks/ipc/api/IPCPerformanceCounters.java |   66 +
 .../ipc/api/IPayloadSerializerDeserializer.java |   27 +
 .../hyracks/ipc/api/IResponseCallback.java      |   19 +
 .../apache/hyracks/ipc/api/RPCInterface.java    |   98 ++
 .../hyracks/ipc/exceptions/IPCException.java    |   35 +
 .../apache/hyracks/ipc/impl/HandleState.java    |   24 +
 .../hyracks/ipc/impl/IPCConnectionManager.java  |  343 +++++
 .../org/apache/hyracks/ipc/impl/IPCHandle.java  |  191 +++
 .../org/apache/hyracks/ipc/impl/IPCSystem.java  |  101 ++
 ...ationBasedPayloadSerializerDeserializer.java |   67 +
 .../org/apache/hyracks/ipc/impl/Message.java    |  127 ++
 .../edu/uci/ics/hyracks/ipc/tests/IPCTest.java  |   92 --
 .../org/apache/hyracks/ipc/tests/IPCTest.java   |   92 ++
 .../maven/plugin/AbstractHyracksMojo.java       |  104 --
 .../maven/plugin/AbstractHyracksServerMojo.java |   30 -
 .../maven/plugin/HyracksCCStartMojo.java        |   50 -
 .../maven/plugin/HyracksNCStartMojo.java        |   71 -
 .../maven/plugin/HyracksServiceRegistry.java    |   48 -
 .../maven/plugin/HyracksStopServicesMojo.java   |   29 -
 .../maven/plugin/AbstractHyracksMojo.java       |  104 ++
 .../maven/plugin/AbstractHyracksServerMojo.java |   30 +
 .../maven/plugin/HyracksCCStartMojo.java        |   50 +
 .../maven/plugin/HyracksNCStartMojo.java        |   71 +
 .../maven/plugin/HyracksServiceRegistry.java    |   48 +
 .../maven/plugin/HyracksStopServicesMojo.java   |   29 +
 .../hyracks/net/buffers/IBufferAcceptor.java    |   31 -
 .../net/buffers/ICloseableBufferAcceptor.java   |   36 -
 .../hyracks/net/exceptions/NetException.java    |   34 -
 .../protocols/muxdemux/ChannelControlBlock.java |  401 ------
 .../net/protocols/muxdemux/ChannelSet.java      |  235 ----
 .../net/protocols/muxdemux/IBufferFactory.java  |   26 -
 .../muxdemux/IChannelOpenListener.java          |   30 -
 .../muxdemux/IChannelReadInterface.java         |   55 -
 .../muxdemux/IChannelWriteInterface.java        |   55 -
 .../net/protocols/muxdemux/IEventCounter.java   |   21 -
 .../muxdemux/MultiplexedConnection.java         |  413 ------
 .../net/protocols/muxdemux/MuxDemux.java        |  162 ---
 .../net/protocols/muxdemux/MuxDemuxCommand.java |   89 --
 .../muxdemux/MuxDemuxPerformanceCounters.java   |   66 -
 .../tcp/ITCPConnectionEventListener.java        |   26 -
 .../protocols/tcp/ITCPConnectionListener.java   |   25 -
 .../net/protocols/tcp/TCPConnection.java        |   93 --
 .../hyracks/net/protocols/tcp/TCPEndpoint.java  |  241 ----
 .../hyracks/net/buffers/IBufferAcceptor.java    |   31 +
 .../net/buffers/ICloseableBufferAcceptor.java   |   36 +
 .../hyracks/net/exceptions/NetException.java    |   34 +
 .../protocols/muxdemux/ChannelControlBlock.java |  401 ++++++
 .../net/protocols/muxdemux/ChannelSet.java      |  235 ++++
 .../net/protocols/muxdemux/IBufferFactory.java  |   26 +
 .../muxdemux/IChannelOpenListener.java          |   30 +
 .../muxdemux/IChannelReadInterface.java         |   55 +
 .../muxdemux/IChannelWriteInterface.java        |   55 +
 .../net/protocols/muxdemux/IEventCounter.java   |   21 +
 .../muxdemux/MultiplexedConnection.java         |  413 ++++++
 .../net/protocols/muxdemux/MuxDemux.java        |  162 +++
 .../net/protocols/muxdemux/MuxDemuxCommand.java |   89 ++
 .../muxdemux/MuxDemuxPerformanceCounters.java   |   66 +
 .../tcp/ITCPConnectionEventListener.java        |   26 +
 .../protocols/tcp/ITCPConnectionListener.java   |   25 +
 .../net/protocols/tcp/TCPConnection.java        |   93 ++
 .../hyracks/net/protocols/tcp/TCPEndpoint.java  |  241 ++++
 .../edu/uci/ics/hyracks/net/tests/NetTest.java  |  213 ---
 .../org/apache/hyracks/net/tests/NetTest.java   |  213 +++
 .../server/drivers/VirtualClusterDriver.java    |   78 --
 .../server/process/HyracksCCProcess.java        |   38 -
 .../server/process/HyracksNCProcess.java        |   38 -
 .../server/process/HyracksServerProcess.java    |   85 --
 .../server/drivers/VirtualClusterDriver.java    |   78 ++
 .../server/process/HyracksCCProcess.java        |   38 +
 .../server/process/HyracksNCProcess.java        |   38 +
 .../server/process/HyracksServerProcess.java    |   85 ++
 .../am/bloomfilter/impls/BloomCalculations.java |  163 ---
 .../am/bloomfilter/impls/BloomFilter.java       |  300 ----
 .../bloomfilter/impls/BloomFilterFactory.java   |   41 -
 .../impls/BloomFilterSpecification.java         |   34 -
 .../am/bloomfilter/impls/MurmurHash128Bit.java  |  256 ----
 .../am/bloomfilter/impls/BloomCalculations.java |  163 +++
 .../am/bloomfilter/impls/BloomFilter.java       |  300 ++++
 .../bloomfilter/impls/BloomFilterFactory.java   |   41 +
 .../impls/BloomFilterSpecification.java         |   34 +
 .../am/bloomfilter/impls/MurmurHash128Bit.java  |  256 ++++
 .../storage/am/btree/api/IBTreeFrame.java       |   36 -
 .../am/btree/api/IBTreeInteriorFrame.java       |   31 -
 .../storage/am/btree/api/IBTreeLeafFrame.java   |   48 -
 .../am/btree/api/IPrefixSlotManager.java        |   82 --
 .../storage/am/btree/api/ITupleAcceptor.java    |   22 -
 .../compressors/FieldPrefixCompressor.java      |  455 ------
 .../am/btree/dataflow/BTreeDataflowHelper.java  |   46 -
 .../dataflow/BTreeDataflowHelperFactory.java    |   38 -
 .../dataflow/BTreeSearchOperatorDescriptor.java |   72 -
 .../BTreeSearchOperatorNodePushable.java        |   82 --
 .../BTreeUpdateSearchOperatorDescriptor.java    |   57 -
 .../BTreeUpdateSearchOperatorNodePushable.java  |   70 -
 .../am/btree/exceptions/BTreeException.java     |   31 -
 .../exceptions/BTreeNotUpdateableException.java |   28 -
 .../frames/BTreeFieldPrefixNSMLeafFrame.java    |  777 -----------
 .../BTreeFieldPrefixNSMLeafFrameFactory.java    |   41 -
 .../am/btree/frames/BTreeLeafFrameType.java     |   21 -
 .../am/btree/frames/BTreeNSMInteriorFrame.java  |  463 -------
 .../frames/BTreeNSMInteriorFrameFactory.java    |   41 -
 .../am/btree/frames/BTreeNSMLeafFrame.java      |  298 ----
 .../btree/frames/BTreeNSMLeafFrameFactory.java  |   41 -
 .../am/btree/frames/OrderedSlotManager.java     |  118 --
 .../hyracks/storage/am/btree/impls/BTree.java   | 1118 ---------------
 .../btree/impls/BTreeCountingSearchCursor.java  |  263 ----
 .../am/btree/impls/BTreeCursorInitialState.java |   77 --
 .../storage/am/btree/impls/BTreeOpContext.java  |  226 ---
 .../am/btree/impls/BTreeRangeSearchCursor.java  |  317 -----
 .../storage/am/btree/impls/BTreeSplitKey.java   |   92 --
 .../impls/FieldPrefixPrefixTupleReference.java  |   42 -
 .../am/btree/impls/FieldPrefixSlotManager.java  |  296 ----
 .../btree/impls/FieldPrefixTupleReference.java  |  133 --
 .../storage/am/btree/impls/NodeFrontier.java    |   29 -
 .../storage/am/btree/impls/RangePredicate.java  |  100 --
 .../btree/impls/UnconditionalTupleAcceptor.java |   28 -
 .../storage/am/btree/util/BTreeUtils.java       |   91 --
 .../storage/am/btree/api/IBTreeFrame.java       |   36 +
 .../am/btree/api/IBTreeInteriorFrame.java       |   31 +
 .../storage/am/btree/api/IBTreeLeafFrame.java   |   48 +
 .../am/btree/api/IPrefixSlotManager.java        |   82 ++
 .../storage/am/btree/api/ITupleAcceptor.java    |   22 +
 .../compressors/FieldPrefixCompressor.java      |  455 ++++++
 .../am/btree/dataflow/BTreeDataflowHelper.java  |   46 +
 .../dataflow/BTreeDataflowHelperFactory.java    |   38 +
 .../dataflow/BTreeSearchOperatorDescriptor.java |   72 +
 .../BTreeSearchOperatorNodePushable.java        |   82 ++
 .../BTreeUpdateSearchOperatorDescriptor.java    |   57 +
 .../BTreeUpdateSearchOperatorNodePushable.java  |   70 +
 .../am/btree/exceptions/BTreeException.java     |   31 +
 .../exceptions/BTreeNotUpdateableException.java |   28 +
 .../frames/BTreeFieldPrefixNSMLeafFrame.java    |  777 +++++++++++
 .../BTreeFieldPrefixNSMLeafFrameFactory.java    |   41 +
 .../am/btree/frames/BTreeLeafFrameType.java     |   21 +
 .../am/btree/frames/BTreeNSMInteriorFrame.java  |  463 +++++++
 .../frames/BTreeNSMInteriorFrameFactory.java    |   41 +
 .../am/btree/frames/BTreeNSMLeafFrame.java      |  298 ++++
 .../btree/frames/BTreeNSMLeafFrameFactory.java  |   41 +
 .../am/btree/frames/OrderedSlotManager.java     |  118 ++
 .../hyracks/storage/am/btree/impls/BTree.java   | 1118 +++++++++++++++
 .../btree/impls/BTreeCountingSearchCursor.java  |  263 ++++
 .../am/btree/impls/BTreeCursorInitialState.java |   77 ++
 .../storage/am/btree/impls/BTreeOpContext.java  |  226 +++
 .../am/btree/impls/BTreeRangeSearchCursor.java  |  317 +++++
 .../storage/am/btree/impls/BTreeSplitKey.java   |   92 ++
 .../impls/FieldPrefixPrefixTupleReference.java  |   42 +
 .../am/btree/impls/FieldPrefixSlotManager.java  |  296 ++++
 .../btree/impls/FieldPrefixTupleReference.java  |  133 ++
 .../storage/am/btree/impls/NodeFrontier.java    |   29 +
 .../storage/am/btree/impls/RangePredicate.java  |  100 ++
 .../btree/impls/UnconditionalTupleAcceptor.java |   28 +
 .../storage/am/btree/util/BTreeUtils.java       |   91 ++
 .../am/common/api/ICursorInitialState.java      |   33 -
 .../storage/am/common/api/IFreePageManager.java |   49 -
 .../am/common/api/IFreePageManagerFactory.java  |   19 -
 .../hyracks/storage/am/common/api/IIndex.java   |  131 --
 .../storage/am/common/api/IIndexAccessor.java   |  101 --
 .../storage/am/common/api/IIndexBulkLoader.java |   42 -
 .../storage/am/common/api/IIndexCursor.java     |   34 -
 .../am/common/api/IIndexDataflowHelper.java     |   37 -
 .../am/common/api/IIndexLifecycleManager.java   |   34 -
 .../api/IIndexLifecycleManagerProvider.java     |   23 -
 .../am/common/api/IIndexOperationContext.java   |   26 -
 .../api/IModificationOperationCallback.java     |   50 -
 .../IModificationOperationCallbackFactory.java  |   25 -
 .../am/common/api/IPrimitiveValueProvider.java  |   20 -
 .../api/IPrimitiveValueProviderFactory.java     |   22 -
 .../am/common/api/ISearchOperationCallback.java |   69 -
 .../api/ISearchOperationCallbackFactory.java    |   25 -
 .../storage/am/common/api/ISearchPredicate.java |   26 -
 .../storage/am/common/api/ISlotManager.java     |   47 -
 .../storage/am/common/api/ISplitKey.java        |   39 -
 .../storage/am/common/api/ITreeIndex.java       |   61 -
 .../am/common/api/ITreeIndexAccessor.java       |   45 -
 .../storage/am/common/api/ITreeIndexCursor.java |   34 -
 .../storage/am/common/api/ITreeIndexFrame.java  |  101 --
 .../common/api/ITreeIndexFrameCompressor.java   |   22 -
 .../am/common/api/ITreeIndexFrameFactory.java   |   22 -
 .../am/common/api/ITreeIndexMetaDataFrame.java  |   61 -
 .../api/ITreeIndexMetaDataFrameFactory.java     |   20 -
 .../am/common/api/ITreeIndexTupleReference.java |   32 -
 .../am/common/api/ITreeIndexTupleWriter.java    |   42 -
 .../api/ITreeIndexTupleWriterFactory.java       |   22 -
 .../storage/am/common/api/ITupleFilter.java     |   22 -
 .../am/common/api/ITupleFilterFactory.java      |   24 -
 .../storage/am/common/api/ITupleUpdater.java    |   29 -
 .../am/common/api/ITupleUpdaterFactory.java     |   22 -
 .../am/common/api/ITwoPCIndexBulkLoader.java    |   39 -
 .../am/common/api/IVirtualFreePageManager.java  |   21 -
 .../storage/am/common/api/IndexException.java   |   28 -
 .../am/common/api/TreeIndexException.java       |   38 -
 .../am/common/api/UnsortedInputException.java   |   31 -
 .../PointablePrimitiveValueProviderFactory.java |   47 -
 .../AbstractIndexOperatorDescriptor.java        |  131 --
 .../AbstractTreeIndexOperatorDescriptor.java    |   77 --
 .../dataflow/IIndexDataflowHelperFactory.java   |   26 -
 .../dataflow/IIndexOperatorDescriptor.java      |   53 -
 .../dataflow/ITreeIndexOperatorDescriptor.java  |   28 -
 .../IndexBulkLoadOperatorNodePushable.java      |  115 --
 .../IndexCreateOperatorNodePushable.java        |   54 -
 .../am/common/dataflow/IndexDataflowHelper.java |  162 ---
 .../dataflow/IndexDropOperatorDescriptor.java   |   53 -
 .../dataflow/IndexDropOperatorNodePushable.java |   55 -
 ...xInsertUpdateDeleteOperatorNodePushable.java |  157 ---
 .../common/dataflow/IndexLifecycleManager.java  |  234 ----
 .../am/common/dataflow/IndexRegistry.java       |   39 -
 .../IndexSearchOperatorNodePushable.java        |  208 ---
 .../TreeIndexBulkLoadOperatorDescriptor.java    |   64 -
 .../TreeIndexCreateOperatorDescriptor.java      |   51 -
 .../dataflow/TreeIndexDataflowHelper.java       |   34 -
 ...reeIndexDiskOrderScanOperatorDescriptor.java |   49 -
 ...eIndexDiskOrderScanOperatorNodePushable.java |   99 --
 ...dexInsertUpdateDeleteOperatorDescriptor.java |   61 -
 .../TreeIndexStatsOperatorDescriptor.java       |   54 -
 .../TreeIndexStatsOperatorNodePushable.java     |   91 --
 .../am/common/datagen/DataGenThread.java        |   99 --
 .../storage/am/common/datagen/DataGenUtils.java |   61 -
 .../DocumentStringFieldValueGenerator.java      |   99 --
 .../datagen/DoubleFieldValueGenerator.java      |   35 -
 .../datagen/FloatFieldValueGenerator.java       |   35 -
 .../am/common/datagen/IFieldValueGenerator.java |   21 -
 .../datagen/IntegerFieldValueGenerator.java     |   35 -
 .../datagen/PersonNameFieldValueGenerator.java  |   97 --
 .../am/common/datagen/ProbabilityHelper.java    |   64 -
 .../SortedDoubleFieldValueGenerator.java        |   41 -
 .../datagen/SortedFloatFieldValueGenerator.java |   41 -
 .../SortedIntegerFieldValueGenerator.java       |   41 -
 .../datagen/StringFieldValueGenerator.java      |   46 -
 .../storage/am/common/datagen/TupleBatch.java   |   51 -
 .../am/common/datagen/TupleGenerator.java       |   80 --
 .../TreeIndexDuplicateKeyException.java         |   30 -
 .../TreeIndexNonExistentKeyException.java       |   31 -
 .../am/common/frames/AbstractSlotManager.java   |   74 -
 .../am/common/frames/FrameOpSpaceStatus.java    |   20 -
 .../am/common/frames/LIFOMetaDataFrame.java     |  161 ---
 .../common/frames/LIFOMetaDataFrameFactory.java |   26 -
 .../am/common/frames/TreeIndexNSMFrame.java     |  301 ----
 .../freepage/LinkedListFreePageManager.java     |  230 ---
 .../LinkedListFreePageManagerFactory.java       |   37 -
 .../common/impls/AbstractSearchPredicate.java   |   44 -
 .../am/common/impls/AbstractTreeIndex.java      |  387 ------
 .../am/common/impls/NoOpOperationCallback.java  |   58 -
 .../impls/NoOpOperationCallbackFactory.java     |   40 -
 .../storage/am/common/impls/NodeFrontier.java   |   29 -
 .../impls/TreeIndexDiskOrderScanCursor.java     |  155 ---
 .../am/common/ophelpers/DoubleArrayList.java    |   98 --
 .../am/common/ophelpers/FindTupleMode.java      |   20 -
 .../ophelpers/FindTupleNoExactMatchPolicy.java  |   20 -
 .../storage/am/common/ophelpers/IndexOp.java    |   20 -
 .../am/common/ophelpers/IndexOperation.java     |   31 -
 .../am/common/ophelpers/IntArrayList.java       |   98 --
 .../am/common/ophelpers/LongArrayList.java      |   99 --
 .../am/common/ophelpers/MultiComparator.java    |  129 --
 .../am/common/ophelpers/SingleComparator.java   |   38 -
 .../am/common/ophelpers/SlotOffTupleOff.java    |   38 -
 .../tuples/ConcatenatingTupleReference.java     |  114 --
 .../am/common/tuples/DualTupleReference.java    |   56 -
 .../tuples/PermutingFrameTupleReference.java    |   65 -
 .../common/tuples/PermutingTupleReference.java  |   52 -
 .../am/common/tuples/SimpleTupleReference.java  |   99 --
 .../am/common/tuples/SimpleTupleWriter.java     |  121 --
 .../common/tuples/SimpleTupleWriterFactory.java |   30 -
 .../common/tuples/TypeAwareTupleReference.java  |  125 --
 .../am/common/tuples/TypeAwareTupleWriter.java  |  155 ---
 .../tuples/TypeAwareTupleWriterFactory.java     |   37 -
 .../common/tuples/VarLenIntEncoderDecoder.java  |   88 --
 .../storage/am/common/util/HashMultiSet.java    |  113 --
 .../am/common/util/IndexFileNameUtil.java       |   27 -
 .../common/util/TreeIndexBufferCacheWarmup.java |  102 --
 .../storage/am/common/util/TreeIndexStats.java  |  161 ---
 .../am/common/util/TreeIndexStatsGatherer.java  |   87 --
 .../storage/am/common/util/TreeIndexUtils.java  |   39 -
 .../am/common/api/ICursorInitialState.java      |   33 +
 .../storage/am/common/api/IFreePageManager.java |   49 +
 .../am/common/api/IFreePageManagerFactory.java  |   19 +
 .../hyracks/storage/am/common/api/IIndex.java   |  131 ++
 .../storage/am/common/api/IIndexAccessor.java   |  101 ++
 .../storage/am/common/api/IIndexBulkLoader.java |   42 +
 .../storage/am/common/api/IIndexCursor.java     |   34 +
 .../am/common/api/IIndexDataflowHelper.java     |   37 +
 .../am/common/api/IIndexLifecycleManager.java   |   34 +
 .../api/IIndexLifecycleManagerProvider.java     |   23 +
 .../am/common/api/IIndexOperationContext.java   |   26 +
 .../api/IModificationOperationCallback.java     |   50 +
 .../IModificationOperationCallbackFactory.java  |   25 +
 .../am/common/api/IPrimitiveValueProvider.java  |   20 +
 .../api/IPrimitiveValueProviderFactory.java     |   22 +
 .../am/common/api/ISearchOperationCallback.java |   69 +
 .../api/ISearchOperationCallbackFactory.java    |   25 +
 .../storage/am/common/api/ISearchPredicate.java |   26 +
 .../storage/am/common/api/ISlotManager.java     |   47 +
 .../storage/am/common/api/ISplitKey.java        |   39 +
 .../storage/am/common/api/ITreeIndex.java       |   61 +
 .../am/common/api/ITreeIndexAccessor.java       |   45 +
 .../storage/am/common/api/ITreeIndexCursor.java |   34 +
 .../storage/am/common/api/ITreeIndexFrame.java  |  101 ++
 .../common/api/ITreeIndexFrameCompressor.java   |   22 +
 .../am/common/api/ITreeIndexFrameFactory.java   |   22 +
 .../am/common/api/ITreeIndexMetaDataFrame.java  |   61 +
 .../api/ITreeIndexMetaDataFrameFactory.java     |   20 +
 .../am/common/api/ITreeIndexTupleReference.java |   32 +
 .../am/common/api/ITreeIndexTupleWriter.java    |   42 +
 .../api/ITreeIndexTupleWriterFactory.java       |   22 +
 .../storage/am/common/api/ITupleFilter.java     |   22 +
 .../am/common/api/ITupleFilterFactory.java      |   24 +
 .../storage/am/common/api/ITupleUpdater.java    |   29 +
 .../am/common/api/ITupleUpdaterFactory.java     |   22 +
 .../am/common/api/ITwoPCIndexBulkLoader.java    |   39 +
 .../am/common/api/IVirtualFreePageManager.java  |   21 +
 .../storage/am/common/api/IndexException.java   |   28 +
 .../am/common/api/TreeIndexException.java       |   38 +
 .../am/common/api/UnsortedInputException.java   |   31 +
 .../PointablePrimitiveValueProviderFactory.java |   47 +
 .../AbstractIndexOperatorDescriptor.java        |  131 ++
 .../AbstractTreeIndexOperatorDescriptor.java    |   77 ++
 .../dataflow/IIndexDataflowHelperFactory.java   |   26 +
 .../dataflow/IIndexOperatorDescriptor.java      |   53 +
 .../dataflow/ITreeIndexOperatorDescriptor.java  |   28 +
 .../IndexBulkLoadOperatorNodePushable.java      |  115 ++
 .../IndexCreateOperatorNodePushable.java        |   54 +
 .../am/common/dataflow/IndexDataflowHelper.java |  162 +++
 .../dataflow/IndexDropOperatorDescriptor.java   |   53 +
 .../dataflow/IndexDropOperatorNodePushable.java |   55 +
 ...xInsertUpdateDeleteOperatorNodePushable.java |  157 +++
 .../common/dataflow/IndexLifecycleManager.java  |  234 ++++
 .../am/common/dataflow/IndexRegistry.java       |   39 +
 .../IndexSearchOperatorNodePushable.java        |  208 +++
 .../TreeIndexBulkLoadOperatorDescriptor.java    |   64 +
 .../TreeIndexCreateOperatorDescriptor.java      |   51 +
 .../dataflow/TreeIndexDataflowHelper.java       |   34 +
 ...reeIndexDiskOrderScanOperatorDescriptor.java |   49 +
 ...eIndexDiskOrderScanOperatorNodePushable.java |   99 ++
 ...dexInsertUpdateDeleteOperatorDescriptor.java |   61 +
 .../TreeIndexStatsOperatorDescriptor.java       |   54 +
 .../TreeIndexStatsOperatorNodePushable.java     |   91 ++
 .../am/common/datagen/DataGenThread.java        |   99 ++
 .../storage/am/common/datagen/DataGenUtils.java |   61 +
 .../DocumentStringFieldValueGenerator.java      |   99 ++
 .../datagen/DoubleFieldValueGenerator.java      |   35 +
 .../datagen/FloatFieldValueGenerator.java       |   35 +
 .../am/common/datagen/IFieldValueGenerator.java |   21 +
 .../datagen/IntegerFieldValueGenerator.java     |   35 +
 .../datagen/PersonNameFieldValueGenerator.java  |   97 ++
 .../am/common/datagen/ProbabilityHelper.java    |   64 +
 .../SortedDoubleFieldValueGenerator.java        |   41 +
 .../datagen/SortedFloatFieldValueGenerator.java |   41 +
 .../SortedIntegerFieldValueGenerator.java       |   41 +
 .../datagen/StringFieldValueGenerator.java      |   46 +
 .../storage/am/common/datagen/TupleBatch.java   |   51 +
 .../am/common/datagen/TupleGenerator.java       |   80 ++
 .../TreeIndexDuplicateKeyException.java         |   30 +
 .../TreeIndexNonExistentKeyException.java       |   31 +
 .../am/common/frames/AbstractSlotManager.java   |   74 +
 .../am/common/frames/FrameOpSpaceStatus.java    |   20 +
 .../am/common/frames/LIFOMetaDataFrame.java     |  161 +++
 .../common/frames/LIFOMetaDataFrameFactory.java |   26 +
 .../am/common/frames/TreeIndexNSMFrame.java     |  301 ++++
 .../freepage/LinkedListFreePageManager.java     |  230 +++
 .../LinkedListFreePageManagerFactory.java       |   37 +
 .../common/impls/AbstractSearchPredicate.java   |   44 +
 .../am/common/impls/AbstractTreeIndex.java      |  387 ++++++
 .../am/common/impls/NoOpOperationCallback.java  |   58 +
 .../impls/NoOpOperationCallbackFactory.java     |   40 +
 .../storage/am/common/impls/NodeFrontier.java   |   29 +
 .../impls/TreeIndexDiskOrderScanCursor.java     |  155 +++
 .../am/common/ophelpers/DoubleArrayList.java    |   98 ++
 .../am/common/ophelpers/FindTupleMode.java      |   20 +
 .../ophelpers/FindTupleNoExactMatchPolicy.java  |   20 +
 .../storage/am/common/ophelpers/IndexOp.java    |   20 +
 .../am/common/ophelpers/IndexOperation.java     |   31 +
 .../am/common/ophelpers/IntArrayList.java       |   98 ++
 .../am/common/ophelpers/LongArrayList.java      |   99 ++
 .../am/common/ophelpers/MultiComparator.java    |  129 ++
 .../am/common/ophelpers/SingleComparator.java   |   38 +
 .../am/common/ophelpers/SlotOffTupleOff.java    |   38 +
 .../tuples/ConcatenatingTupleReference.java     |  114 ++
 .../am/common/tuples/DualTupleReference.java    |   56 +
 .../tuples/PermutingFrameTupleReference.java    |   65 +
 .../common/tuples/PermutingTupleReference.java  |   52 +
 .../am/common/tuples/SimpleTupleReference.java  |   99 ++
 .../am/common/tuples/SimpleTupleWriter.java     |  121 ++
 .../common/tuples/SimpleTupleWriterFactory.java |   30 +
 .../common/tuples/TypeAwareTupleReference.java  |  125 ++
 .../am/common/tuples/TypeAwareTupleWriter.java  |  155 +++
 .../tuples/TypeAwareTupleWriterFactory.java     |   37 +
 .../common/tuples/VarLenIntEncoderDecoder.java  |   88 ++
 .../storage/am/common/util/HashMultiSet.java    |  113 ++
 .../am/common/util/IndexFileNameUtil.java       |   27 +
 .../common/util/TreeIndexBufferCacheWarmup.java |  102 ++
 .../storage/am/common/util/TreeIndexStats.java  |  161 +++
 .../am/common/util/TreeIndexStatsGatherer.java  |   87 ++
 .../storage/am/common/util/TreeIndexUtils.java  |   39 +
 .../dataflow/ExternalBTreeDataflowHelper.java   |   88 --
 .../ExternalBTreeDataflowHelperFactory.java     |   51 -
 .../ExternalBTreeWithBuddyDataflowHelper.java   |   89 --
 ...rnalBTreeWithBuddyDataflowHelperFactory.java |   52 -
 .../btree/dataflow/LSMBTreeDataflowHelper.java  |   73 -
 .../dataflow/LSMBTreeDataflowHelperFactory.java |   60 -
 .../impls/AntimatterAwareTupleAcceptor.java     |   32 -
 .../am/lsm/btree/impls/ExternalBTree.java       |  708 ----------
 .../lsm/btree/impls/ExternalBTreeOpContext.java |  135 --
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |  897 ------------
 .../impls/ExternalBTreeWithBuddyOpContext.java  |  122 --
 .../storage/am/lsm/btree/impls/LSMBTree.java    |  865 ------------
 .../btree/impls/LSMBTreeCursorInitialState.java |  101 --
 .../lsm/btree/impls/LSMBTreeDiskComponent.java  |   58 -
 .../impls/LSMBTreeDiskComponentFactory.java     |   57 -
 .../am/lsm/btree/impls/LSMBTreeFileManager.java |  221 ---
 .../lsm/btree/impls/LSMBTreeFlushOperation.java |  104 --
 .../btree/impls/LSMBTreeMemoryComponent.java    |   46 -
 .../lsm/btree/impls/LSMBTreeMergeOperation.java |  109 --
 .../am/lsm/btree/impls/LSMBTreeOpContext.java   |  198 ---
 .../btree/impls/LSMBTreePointSearchCursor.java  |  227 ---
 .../btree/impls/LSMBTreeRangeSearchCursor.java  |  212 ---
 .../lsm/btree/impls/LSMBTreeSearchCursor.java   |  136 --
 .../impls/LSMBTreeWithBuddyAbstractCursor.java  |  187 ---
 .../LSMBTreeWithBuddyCursorInitialState.java    |  113 --
 .../impls/LSMBTreeWithBuddyDiskComponent.java   |   69 -
 .../LSMBTreeWithBuddyDiskComponentFactory.java  |   61 -
 .../impls/LSMBTreeWithBuddyFileManager.java     |  257 ----
 .../impls/LSMBTreeWithBuddyMemoryComponent.java |   59 -
 .../impls/LSMBTreeWithBuddyMergeOperation.java  |  132 --
 .../impls/LSMBTreeWithBuddySearchCursor.java    |  120 --
 .../impls/LSMBTreeWithBuddySortedCursor.java    |  165 ---
 .../btree/impls/LSMBuddyBTreeMergeCursor.java   |   77 --
 .../btree/tuples/LSMBTreeCopyTupleWriter.java   |   35 -
 .../tuples/LSMBTreeCopyTupleWriterFactory.java  |   37 -
 .../LSMBTreeRefrencingTupleWriterFactory.java   |   65 -
 .../btree/tuples/LSMBTreeTupleReference.java    |   93 --
 .../lsm/btree/tuples/LSMBTreeTupleWriter.java   |   86 --
 .../tuples/LSMBTreeTupleWriterFactory.java      |   38 -
 .../am/lsm/btree/util/LSMBTreeUtils.java        |  213 ---
 .../dataflow/ExternalBTreeDataflowHelper.java   |   88 ++
 .../ExternalBTreeDataflowHelperFactory.java     |   51 +
 .../ExternalBTreeWithBuddyDataflowHelper.java   |   89 ++
 ...rnalBTreeWithBuddyDataflowHelperFactory.java |   52 +
 .../btree/dataflow/LSMBTreeDataflowHelper.java  |   73 +
 .../dataflow/LSMBTreeDataflowHelperFactory.java |   60 +
 .../impls/AntimatterAwareTupleAcceptor.java     |   32 +
 .../am/lsm/btree/impls/ExternalBTree.java       |  708 ++++++++++
 .../lsm/btree/impls/ExternalBTreeOpContext.java |  135 ++
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |  897 ++++++++++++
 .../impls/ExternalBTreeWithBuddyOpContext.java  |  122 ++
 .../storage/am/lsm/btree/impls/LSMBTree.java    |  865 ++++++++++++
 .../btree/impls/LSMBTreeCursorInitialState.java |  101 ++
 .../lsm/btree/impls/LSMBTreeDiskComponent.java  |   58 +
 .../impls/LSMBTreeDiskComponentFactory.java     |   57 +
 .../am/lsm/btree/impls/LSMBTreeFileManager.java |  221 +++
 .../lsm/btree/impls/LSMBTreeFlushOperation.java |  104 ++
 .../btree/impls/LSMBTreeMemoryComponent.java    |   46 +
 .../lsm/btree/impls/LSMBTreeMergeOperation.java |  109 ++
 .../am/lsm/btree/impls/LSMBTreeOpContext.java   |  198 +++
 .../btree/impls/LSMBTreePointSearchCursor.java  |  227 +++
 .../btree/impls/LSMBTreeRangeSearchCursor.java  |  212 +++
 .../lsm/btree/impls/LSMBTreeSearchCursor.java   |  136 ++
 .../impls/LSMBTreeWithBuddyAbstractCursor.java  |  187 +++
 .../LSMBTreeWithBuddyCursorInitialState.java    |  113 ++
 .../impls/LSMBTreeWithBuddyDiskComponent.java   |   69 +
 .../LSMBTreeWithBuddyDiskComponentFactory.java  |   61 +
 .../impls/LSMBTreeWithBuddyFileManager.java     |  257 ++++
 .../impls/LSMBTreeWithBuddyMemoryComponent.java |   59 +
 .../impls/LSMBTreeWithBuddyMergeOperation.java  |  132 ++
 .../impls/LSMBTreeWithBuddySearchCursor.java    |  120 ++
 .../impls/LSMBTreeWithBuddySortedCursor.java    |  165 +++
 .../btree/impls/LSMBuddyBTreeMergeCursor.java   |   77 ++
 .../btree/tuples/LSMBTreeCopyTupleWriter.java   |   35 +
 .../tuples/LSMBTreeCopyTupleWriterFactory.java  |   37 +
 .../LSMBTreeRefrencingTupleWriterFactory.java   |   65 +
 .../btree/tuples/LSMBTreeTupleReference.java    |   93 ++
 .../lsm/btree/tuples/LSMBTreeTupleWriter.java   |   86 ++
 .../tuples/LSMBTreeTupleWriterFactory.java      |   38 +
 .../am/lsm/btree/util/LSMBTreeUtils.java        |  213 +++
 .../am/lsm/common/api/ILSMComponent.java        |   46 -
 .../am/lsm/common/api/ILSMComponentFactory.java |   27 -
 .../am/lsm/common/api/ILSMComponentFilter.java  |   36 -
 .../common/api/ILSMComponentFilterFactory.java  |   20 -
 .../common/api/ILSMComponentFilterFrame.java    |   40 -
 .../api/ILSMComponentFilterFrameFactory.java    |   19 -
 .../common/api/ILSMComponentFilterManager.java  |   32 -
 .../storage/am/lsm/common/api/ILSMHarness.java  |   63 -
 .../am/lsm/common/api/ILSMIOOperation.java      |   42 -
 .../lsm/common/api/ILSMIOOperationCallback.java |   52 -
 .../api/ILSMIOOperationCallbackFactory.java     |   21 -
 .../api/ILSMIOOperationCallbackProvider.java    |   19 -
 .../common/api/ILSMIOOperationScheduler.java    |   21 -
 .../api/ILSMIOOperationSchedulerProvider.java   |   23 -
 .../storage/am/lsm/common/api/ILSMIndex.java    |   50 -
 .../am/lsm/common/api/ILSMIndexAccessor.java    |  119 --
 .../common/api/ILSMIndexAccessorInternal.java   |   38 -
 .../am/lsm/common/api/ILSMIndexFileManager.java |   77 --
 .../am/lsm/common/api/ILSMIndexInternal.java    |   88 --
 .../common/api/ILSMIndexOperationContext.java   |   40 -
 .../lsm/common/api/ILSMIndexReplicationJob.java |   23 -
 .../am/lsm/common/api/ILSMMergePolicy.java      |   28 -
 .../lsm/common/api/ILSMMergePolicyFactory.java  |   36 -
 .../am/lsm/common/api/ILSMOperationTracker.java |   58 -
 .../api/ILSMOperationTrackerProvider.java       |   23 -
 .../lsm/common/api/ILSMTreeTupleReference.java  |   22 -
 .../storage/am/lsm/common/api/ITwoPCIndex.java  |   73 -
 .../am/lsm/common/api/IVirtualBufferCache.java  |   29 -
 .../common/api/IVirtualBufferCacheProvider.java |   24 -
 .../AbstractLSMIndexDataflowHelper.java         |   73 -
 .../AbstractLSMIndexDataflowHelperFactory.java  |   62 -
 .../LSMIndexCompactOperatorNodePushable.java    |   68 -
 ...xInsertUpdateDeleteOperatorNodePushable.java |  121 --
 .../LSMTreeIndexCompactOperatorDescriptor.java  |   54 -
 ...dexInsertUpdateDeleteOperatorDescriptor.java |   63 -
 .../common/frames/LSMComponentFilterFrame.java  |  104 --
 .../frames/LSMComponentFilterFrameFactory.java  |   36 -
 .../common/freepage/VirtualFreePageManager.java |  116 --
 .../impls/AbortRecoverLSMIndexFileManager.java  |   23 -
 .../common/impls/AbstractDiskLSMComponent.java  |  104 --
 .../lsm/common/impls/AbstractLSMComponent.java  |   44 -
 .../am/lsm/common/impls/AbstractLSMIndex.java   |  333 -----
 .../impls/AbstractLSMIndexFileManager.java      |  429 ------
 .../impls/AbstractMemoryLSMComponent.java       |  204 ---
 .../lsm/common/impls/AsynchronousScheduler.java |  100 --
 .../am/lsm/common/impls/BTreeFactory.java       |   42 -
 .../BlockingIOOperationCallbackWrapper.java     |   62 -
 .../BloomFilterAwareBTreePointSearchCursor.java |   40 -
 .../lsm/common/impls/ConstantMergePolicy.java   |   57 -
 .../impls/ConstantMergePolicyFactory.java       |   57 -
 .../lsm/common/impls/ExternalIndexHarness.java  |  361 -----
 .../am/lsm/common/impls/IndexFactory.java       |   43 -
 .../impls/LSMComponentFileReferences.java       |   48 -
 .../am/lsm/common/impls/LSMComponentFilter.java |  135 --
 .../common/impls/LSMComponentFilterFactory.java |   37 -
 .../common/impls/LSMComponentFilterManager.java |  134 --
 .../am/lsm/common/impls/LSMComponentState.java  |   23 -
 .../storage/am/lsm/common/impls/LSMHarness.java |  496 -------
 .../am/lsm/common/impls/LSMIOOperationTask.java |   33 -
 .../common/impls/LSMIndexReplicationJob.java    |   42 -
 .../lsm/common/impls/LSMIndexSearchCursor.java  |  278 ----
 .../am/lsm/common/impls/LSMOperationType.java   |   24 -
 .../lsm/common/impls/LSMTreeIndexAccessor.java  |  161 ---
 .../impls/MultitenantVirtualBufferCache.java    |  159 ---
 .../am/lsm/common/impls/NoMergePolicy.java      |   36 -
 .../lsm/common/impls/NoMergePolicyFactory.java  |   58 -
 .../common/impls/NoOpIOOperationCallback.java   |   59 -
 .../impls/NoOpOperationTrackerProvider.java     |   66 -
 .../am/lsm/common/impls/PrefixMergePolicy.java  |   92 --
 .../common/impls/PrefixMergePolicyFactory.java  |   59 -
 .../lsm/common/impls/SynchronousScheduler.java  |   33 -
 .../impls/SynchronousSchedulerProvider.java     |   29 -
 .../ThreadCountingOperationTrackerProvider.java |   35 -
 .../lsm/common/impls/ThreadCountingTracker.java |   63 -
 .../am/lsm/common/impls/TreeIndexFactory.java   |   41 -
 .../am/lsm/common/impls/VirtualBufferCache.java |  386 ------
 .../am/lsm/common/api/ILSMComponent.java        |   46 +
 .../am/lsm/common/api/ILSMComponentFactory.java |   27 +
 .../am/lsm/common/api/ILSMComponentFilter.java  |   36 +
 .../common/api/ILSMComponentFilterFactory.java  |   20 +
 .../common/api/ILSMComponentFilterFrame.java    |   40 +
 .../api/ILSMComponentFilterFrameFactory.java    |   19 +
 .../common/api/ILSMComponentFilterManager.java  |   32 +
 .../storage/am/lsm/common/api/ILSMHarness.java  |   63 +
 .../am/lsm/common/api/ILSMIOOperation.java      |   42 +
 .../lsm/common/api/ILSMIOOperationCallback.java |   52 +
 .../api/ILSMIOOperationCallbackFactory.java     |   21 +
 .../api/ILSMIOOperationCallbackProvider.java    |   19 +
 .../common/api/ILSMIOOperationScheduler.java    |   21 +
 .../api/ILSMIOOperationSchedulerProvider.java   |   23 +
 .../storage/am/lsm/common/api/ILSMIndex.java    |   50 +
 .../am/lsm/common/api/ILSMIndexAccessor.java    |  119 ++
 .../common/api/ILSMIndexAccessorInternal.java   |   38 +
 .../am/lsm/common/api/ILSMIndexFileManager.java |   77 ++
 .../am/lsm/common/api/ILSMIndexInternal.java    |   88 ++
 .../common/api/ILSMIndexOperationContext.java   |   40 +
 .../lsm/common/api/ILSMIndexReplicationJob.java |   23 +
 .../am/lsm/common/api/ILSMMergePolicy.java      |   28 +
 .../lsm/common/api/ILSMMergePolicyFactory.java  |   36 +
 .../am/lsm/common/api/ILSMOperationTracker.java |   58 +
 .../api/ILSMOperationTrackerProvider.java       |   23 +
 .../lsm/common/api/ILSMTreeTupleReference.java  |   22 +
 .../storage/am/lsm/common/api/ITwoPCIndex.java  |   73 +
 .../am/lsm/common/api/IVirtualBufferCache.java  |   29 +
 .../common/api/IVirtualBufferCacheProvider.java |   24 +
 .../AbstractLSMIndexDataflowHelper.java         |   73 +
 .../AbstractLSMIndexDataflowHelperFactory.java  |   62 +
 .../LSMIndexCompactOperatorNodePushable.java    |   68 +
 ...xInsertUpdateDeleteOperatorNodePushable.java |  121 ++
 .../LSMTreeIndexCompactOperatorDescriptor.java  |   54 +
 ...dexInsertUpdateDeleteOperatorDescriptor.java |   63 +
 .../common/frames/LSMComponentFilterFrame.java  |  104 ++
 .../frames/LSMComponentFilterFrameFactory.java  |   36 +
 .../common/freepage/VirtualFreePageManager.java |  116 ++
 .../impls/AbortRecoverLSMIndexFileManager.java  |   23 +
 .../common/impls/AbstractDiskLSMComponent.java  |  104 ++
 .../lsm/common/impls/AbstractLSMComponent.java  |   44 +
 .../am/lsm/common/impls/AbstractLSMIndex.java   |  333 +++++
 .../impls/AbstractLSMIndexFileManager.java      |  429 ++++++
 .../impls/AbstractMemoryLSMComponent.java       |  204 +++
 .../lsm/common/impls/AsynchronousScheduler.java |  100 ++
 .../am/lsm/common/impls/BTreeFactory.java       |   42 +
 .../BlockingIOOperationCallbackWrapper.java     |   62 +
 .../BloomFilterAwareBTreePointSearchCursor.java |   40 +
 .../lsm/common/impls/ConstantMergePolicy.java   |   57 +
 .../impls/ConstantMergePolicyFactory.java       |   57 +
 .../lsm/common/impls/ExternalIndexHarness.java  |  361 +++++
 .../am/lsm/common/impls/IndexFactory.java       |   43 +
 .../impls/LSMComponentFileReferences.java       |   48 +
 .../am/lsm/common/impls/LSMComponentFilter.java |  135 ++
 .../common/impls/LSMComponentFilterFactory.java |   37 +
 .../common/impls/LSMComponentFilterManager.java |  134 ++
 .../am/lsm/common/impls/LSMComponentState.java  |   23 +
 .../storage/am/lsm/common/impls/LSMHarness.java |  496 +++++++
 .../am/lsm/common/impls/LSMIOOperationTask.java |   33 +
 .../common/impls/LSMIndexReplicationJob.java    |   42 +
 .../lsm/common/impls/LSMIndexSearchCursor.java  |  278 ++++
 .../am/lsm/common/impls/LSMOperationType.java   |   24 +
 .../lsm/common/impls/LSMTreeIndexAccessor.java  |  161 +++
 .../impls/MultitenantVirtualBufferCache.java    |  159 +++
 .../am/lsm/common/impls/NoMergePolicy.java      |   36 +
 .../lsm/common/impls/NoMergePolicyFactory.java  |   58 +
 .../common/impls/NoOpIOOperationCallback.java   |   59 +
 .../impls/NoOpOperationTrackerProvider.java     |   66 +
 .../am/lsm/common/impls/PrefixMergePolicy.java  |   92 ++
 .../common/impls/PrefixMergePolicyFactory.java  |   59 +
 .../lsm/common/impls/SynchronousScheduler.java  |   33 +
 .../impls/SynchronousSchedulerProvider.java     |   29 +
 .../ThreadCountingOperationTrackerProvider.java |   35 +
 .../lsm/common/impls/ThreadCountingTracker.java |   63 +
 .../am/lsm/common/impls/TreeIndexFactory.java   |   41 +
 .../am/lsm/common/impls/VirtualBufferCache.java |  386 ++++++
 .../lsm/invertedindex/api/IInvertedIndex.java   |   39 -
 .../api/IInvertedIndexAccessor.java             |   35 -
 .../api/IInvertedIndexFileNameMapper.java       |   23 -
 .../api/IInvertedIndexOperatorDescriptor.java   |   34 -
 .../api/IInvertedIndexSearchModifier.java       |   26 -
 .../IInvertedIndexSearchModifierFactory.java    |   22 -
 .../api/IInvertedIndexSearcher.java             |   42 -
 .../invertedindex/api/IInvertedListBuilder.java |   32 -
 .../api/IInvertedListBuilderFactory.java        |   20 -
 .../invertedindex/api/IInvertedListCursor.java  |   54 -
 .../lsm/invertedindex/api/IObjectFactory.java   |   20 -
 .../api/IPartitionedInvertedIndex.java          |   31 -
 ...tractLSMInvertedIndexOperatorDescriptor.java |   90 --
 .../BinaryTokenizerOperatorDescriptor.java      |   64 -
 .../BinaryTokenizerOperatorNodePushable.java    |  161 ---
 ...InvertedIndexBulkLoadOperatorDescriptor.java |   67 -
 .../LSMInvertedIndexCompactOperator.java        |   55 -
 ...SMInvertedIndexCreateOperatorDescriptor.java |   57 -
 .../LSMInvertedIndexDataflowHelper.java         |   90 --
 .../LSMInvertedIndexDataflowHelperFactory.java  |   65 -
 ...InvertedIndexInsertUpdateDeleteOperator.java |   65 -
 ...SMInvertedIndexSearchOperatorDescriptor.java |   75 -
 ...InvertedIndexSearchOperatorNodePushable.java |   74 -
 ...rtitionedLSMInvertedIndexDataflowHelper.java |   90 --
 ...edLSMInvertedIndexDataflowHelperFactory.java |   64 -
 .../exceptions/InvertedIndexException.java      |   30 -
 .../OccurrenceThresholdPanicException.java      |   26 -
 .../invertedindex/impls/LSMInvertedIndex.java   |  957 -------------
 .../impls/LSMInvertedIndexAccessor.java         |  183 ---
 ...nvertedIndexDeletedKeysBTreeMergeCursor.java |   63 -
 .../impls/LSMInvertedIndexDiskComponent.java    |   73 -
 .../LSMInvertedIndexDiskComponentFactory.java   |   58 -
 .../impls/LSMInvertedIndexFileManager.java      |  211 ---
 .../impls/LSMInvertedIndexFlushOperation.java   |  108 --
 .../impls/LSMInvertedIndexMemoryComponent.java  |   57 -
 .../impls/LSMInvertedIndexMergeOperation.java   |  118 --
 .../impls/LSMInvertedIndexOpContext.java        |  165 ---
 .../LSMInvertedIndexRangeSearchCursor.java      |  117 --
 ...ertedIndexRangeSearchCursorInitialState.java |  132 --
 .../impls/LSMInvertedIndexSearchCursor.java     |  190 ---
 ...SMInvertedIndexSearchCursorInitialState.java |  128 --
 .../impls/PartitionedLSMInvertedIndex.java      |   72 -
 .../inmemory/InMemoryInvertedIndex.java         |  210 ---
 .../inmemory/InMemoryInvertedIndexAccessor.java |  117 --
 .../InMemoryInvertedIndexOpContext.java         |   94 --
 .../inmemory/InMemoryInvertedListCursor.java    |  240 ----
 .../PartitionedInMemoryInvertedIndex.java       |  147 --
 ...artitionedInMemoryInvertedIndexAccessor.java |   33 -
 ...rtitionedInMemoryInvertedIndexOpContext.java |   36 -
 .../FixedSizeElementInvertedListBuilder.java    |   80 --
 ...edSizeElementInvertedListBuilderFactory.java |   34 -
 .../FixedSizeElementInvertedListCursor.java     |  295 ----
 .../ondisk/FixedSizeFrameTupleAccessor.java     |  108 --
 .../ondisk/FixedSizeFrameTupleAppender.java     |  127 --
 .../ondisk/FixedSizeTupleReference.java         |   61 -
 .../ondisk/OnDiskInvertedIndex.java             |  685 ---------
 .../ondisk/OnDiskInvertedIndexFactory.java      |   61 -
 .../ondisk/OnDiskInvertedIndexOpContext.java    |   60 -
 .../OnDiskInvertedIndexRangeSearchCursor.java   |  138 --
 .../ondisk/OnDiskInvertedIndexSearchCursor.java |  108 --
 .../ondisk/PartitionedOnDiskInvertedIndex.java  |  115 --
 .../PartitionedOnDiskInvertedIndexFactory.java  |   48 -
 .../search/AbstractTOccurrenceSearcher.java     |  153 --
 .../invertedindex/search/ArrayListFactory.java  |   27 -
 .../ConjunctiveEditDistanceSearchModifier.java  |   39 -
 ...nctiveEditDistanceSearchModifierFactory.java |   36 -
 ...njunctiveListEditDistanceSearchModifier.java |   38 -
 ...veListEditDistanceSearchModifierFactory.java |   35 -
 .../search/ConjunctiveSearchModifier.java       |   46 -
 .../ConjunctiveSearchModifierFactory.java       |   28 -
 .../search/EditDistanceSearchModifier.java      |   70 -
 .../EditDistanceSearchModifierFactory.java      |   37 -
 .../search/InvertedIndexSearchPredicate.java    |   79 --
 .../search/InvertedListCursorFactory.java       |   34 -
 .../search/InvertedListMerger.java              |  322 -----
 .../search/InvertedListPartitions.java          |   94 --
 .../search/JaccardSearchModifier.java           |   63 -
 .../search/JaccardSearchModifierFactory.java    |   35 -
 .../search/ListEditDistanceSearchModifier.java  |   60 -
 .../ListEditDistanceSearchModifierFactory.java  |   35 -
 .../search/PartitionedTOccurrenceSearcher.java  |  188 ---
 .../lsm/invertedindex/search/SearchResult.java  |  185 ---
 .../search/TOccurrenceSearcher.java             |   63 -
 .../AbstractUTF8StringBinaryTokenizer.java      |   81 --
 .../tokenizers/AbstractUTF8Token.java           |  101 --
 .../tokenizers/AbstractUTF8TokenFactory.java    |   32 -
 .../DelimitedUTF8StringBinaryTokenizer.java     |   97 --
 ...limitedUTF8StringBinaryTokenizerFactory.java |   38 -
 .../tokenizers/HashedUTF8NGramToken.java        |   60 -
 .../tokenizers/HashedUTF8NGramTokenFactory.java |   34 -
 .../tokenizers/HashedUTF8WordToken.java         |   83 --
 .../tokenizers/HashedUTF8WordTokenFactory.java  |   34 -
 .../tokenizers/IBinaryTokenizer.java            |   29 -
 .../tokenizers/IBinaryTokenizerFactory.java     |   22 -
 .../invertedindex/tokenizers/INGramToken.java   |   24 -
 .../am/lsm/invertedindex/tokenizers/IToken.java |   37 -
 .../invertedindex/tokenizers/ITokenFactory.java |   22 -
 .../lsm/invertedindex/tokenizers/IntArray.java  |   76 -
 .../NGramUTF8StringBinaryTokenizer.java         |  122 --
 .../NGramUTF8StringBinaryTokenizerFactory.java  |   42 -
 .../tokenizers/UTF8NGramToken.java              |   87 --
 .../tokenizers/UTF8NGramTokenFactory.java       |   35 -
 .../invertedindex/tokenizers/UTF8WordToken.java |   47 -
 .../tokenizers/UTF8WordTokenFactory.java        |   35 -
 .../InvertedIndexTokenizingTupleIterator.java   |   78 --
 .../invertedindex/util/InvertedIndexUtils.java  |  220 ---
 .../am/lsm/invertedindex/util/ObjectCache.java  |   51 -
 ...nedInvertedIndexTokenizingTupleIterator.java |   72 -
 .../lsm/invertedindex/api/IInvertedIndex.java   |   39 +
 .../api/IInvertedIndexAccessor.java             |   35 +
 .../api/IInvertedIndexFileNameMapper.java       |   23 +
 .../api/IInvertedIndexOperatorDescriptor.java   |   34 +
 .../api/IInvertedIndexSearchModifier.java       |   26 +
 .../IInvertedIndexSearchModifierFactory.java    |   22 +
 .../api/IInvertedIndexSearcher.java             |   42 +
 .../invertedindex/api/IInvertedListBuilder.java |   32 +
 .../api/IInvertedListBuilderFactory.java        |   20 +
 .../invertedindex/api/IInvertedListCursor.java  |   54 +
 .../lsm/invertedindex/api/IObjectFactory.java   |   20 +
 .../api/IPartitionedInvertedIndex.java          |   31 +
 ...tractLSMInvertedIndexOperatorDescriptor.java |   90 ++
 .../BinaryTokenizerOperatorDescriptor.java      |   64 +
 .../BinaryTokenizerOperatorNodePushable.java    |  161 +++
 ...InvertedIndexBulkLoadOperatorDescriptor.java |   67 +
 .../LSMInvertedIndexCompactOperator.java        |   55 +
 ...SMInvertedIndexCreateOperatorDescriptor.java |   57 +
 .../LSMInvertedIndexDataflowHelper.java         |   90 ++
 .../LSMInvertedIndexDataflowHelperFactory.java  |   65 +
 ...InvertedIndexInsertUpdateDeleteOperator.java |   65 +
 ...SMInvertedIndexSearchOperatorDescriptor.java |   75 +
 ...InvertedIndexSearchOperatorNodePushable.java |   74 +
 ...rtitionedLSMInvertedIndexDataflowHelper.java |   90 ++
 ...edLSMInvertedIndexDataflowHelperFactory.java |   64 +
 .../exceptions/InvertedIndexException.java      |   30 +
 .../OccurrenceThresholdPanicException.java      |   26 +
 .../invertedindex/impls/LSMInvertedIndex.java   |  957 +++++++++++++
 .../impls/LSMInvertedIndexAccessor.java         |  183 +++
 ...nvertedIndexDeletedKeysBTreeMergeCursor.java |   63 +
 .../impls/LSMInvertedIndexDiskComponent.java    |   73 +
 .../LSMInvertedIndexDiskComponentFactory.java   |   58 +
 .../impls/LSMInvertedIndexFileManager.java      |  211 +++
 .../impls/LSMInvertedIndexFlushOperation.java   |  108 ++
 .../impls/LSMInvertedIndexMemoryComponent.java  |   57 +
 .../impls/LSMInvertedIndexMergeOperation.java   |  118 ++
 .../impls/LSMInvertedIndexOpContext.java        |  165 +++
 .../LSMInvertedIndexRangeSearchCursor.java      |  117 ++
 ...ertedIndexRangeSearchCursorInitialState.java |  132 ++
 .../impls/LSMInvertedIndexSearchCursor.java     |  190 +++
 ...SMInvertedIndexSearchCursorInitialState.java |  128 ++
 .../impls/PartitionedLSMInvertedIndex.java      |   72 +
 .../inmemory/InMemoryInvertedIndex.java         |  210 +++
 .../inmemory/InMemoryInvertedIndexAccessor.java |  117 ++
 .../InMemoryInvertedIndexOpContext.java         |   94 ++
 .../inmemory/InMemoryInvertedListCursor.java    |  240 ++++
 .../PartitionedInMemoryInvertedIndex.java       |  147 ++
 ...artitionedInMemoryInvertedIndexAccessor.java |   33 +
 ...rtitionedInMemoryInvertedIndexOpContext.java |   36 +
 .../FixedSizeElementInvertedListBuilder.java    |   80 ++
 ...edSizeElementInvertedListBuilderFactory.java |   34 +
 .../FixedSizeElementInvertedListCursor.java     |  295 ++++
 .../ondisk/FixedSizeFrameTupleAccessor.java     |  108 ++
 .../ondisk/FixedSizeFrameTupleAppender.java     |  127 ++
 .../ondisk/FixedSizeTupleReference.java         |   61 +
 .../ondisk/OnDiskInvertedIndex.java             |  685 +++++++++
 .../ondisk/OnDiskInvertedIndexFactory.java      |   61 +
 .../ondisk/OnDiskInvertedIndexOpContext.java    |   60 +
 .../OnDiskInvertedIndexRangeSearchCursor.java   |  138 ++
 .../ondisk/OnDiskInvertedIndexSearchCursor.java |  108 ++
 .../ondisk/PartitionedOnDiskInvertedIndex.java  |  115 ++
 .../PartitionedOnDiskInvertedIndexFactory.java  |   48 +
 .../search/AbstractTOccurrenceSearcher.java     |  153 ++
 .../invertedindex/search/ArrayListFactory.java  |   27 +
 .../ConjunctiveEditDistanceSearchModifier.java  |   39 +
 ...nctiveEditDistanceSearchModifierFactory.java |   36 +
 ...njunctiveListEditDistanceSearchModifier.java |   38 +
 ...veListEditDistanceSearchModifierFactory.java |   35 +
 .../search/ConjunctiveSearchModifier.java       |   46 +
 .../ConjunctiveSearchModifierFactory.java       |   28 +
 .../search/EditDistanceSearchModifier.java      |   70 +
 .../EditDistanceSearchModifierFactory.java      |   37 +
 .../search/InvertedIndexSearchPredicate.java    |   79 ++
 .../search/InvertedListCursorFactory.java       |   34 +
 .../search/InvertedListMerger.java              |  322 +++++
 .../search/InvertedListPartitions.java          |   94 ++
 .../search/JaccardSearchModifier.java           |   63 +
 .../search/JaccardSearchModifierFactory.java    |   35 +
 .../search/ListEditDistanceSearchModifier.java  |   60 +
 .../ListEditDistanceSearchModifierFactory.java  |   35 +
 .../search/PartitionedTOccurrenceSearcher.java  |  188 +++
 .../lsm/invertedindex/search/SearchResult.java  |  185 +++
 .../search/TOccurrenceSearcher.java             |   63 +
 .../AbstractUTF8StringBinaryTokenizer.java      |   81 ++
 .../tokenizers/AbstractUTF8Token.java           |  101 ++
 .../tokenizers/AbstractUTF8TokenFactory.java    |   32 +
 .../DelimitedUTF8StringBinaryTokenizer.java     |   97 ++
 ...limitedUTF8StringBinaryTokenizerFactory.java |   38 +
 .../tokenizers/HashedUTF8NGramToken.java        |   60 +
 .../tokenizers/HashedUTF8NGramTokenFactory.java |   34 +
 .../tokenizers/HashedUTF8WordToken.java         |   83 ++
 .../tokenizers/HashedUTF8WordTokenFactory.java  |   34 +
 .../tokenizers/IBinaryTokenizer.java            |   29 +
 .../tokenizers/IBinaryTokenizerFactory.java     |   22 +
 .../invertedindex/tokenizers/INGramToken.java   |   24 +
 .../am/lsm/invertedindex/tokenizers/IToken.java |   37 +
 .../invertedindex/tokenizers/ITokenFactory.java |   22 +
 .../lsm/invertedindex/tokenizers/IntArray.java  |   76 +
 .../NGramUTF8StringBinaryTokenizer.java         |  122 ++
 .../NGramUTF8StringBinaryTokenizerFactory.java  |   42 +
 .../tokenizers/UTF8NGramToken.java              |   87 ++
 .../tokenizers/UTF8NGramTokenFactory.java       |   35 +
 .../invertedindex/tokenizers/UTF8WordToken.java |   47 +
 .../tokenizers/UTF8WordTokenFactory.java        |   35 +
 .../InvertedIndexTokenizingTupleIterator.java   |   78 ++
 .../invertedindex/util/InvertedIndexUtils.java  |  220 +++
 .../am/lsm/invertedindex/util/ObjectCache.java  |   51 +
 ...nedInvertedIndexTokenizingTupleIterator.java |   72 +
 .../AbstractLSMRTreeDataflowHelper.java         |   97 --
 .../dataflow/ExternalRTreeDataflowHelper.java   |  112 --
 .../ExternalRTreeDataflowHelperFactory.java     |   57 -
 .../rtree/dataflow/LSMRTreeDataflowHelper.java  |   92 --
 .../dataflow/LSMRTreeDataflowHelperFactory.java |   75 -
 ...RTreeWithAntiMatterTuplesDataflowHelper.java |   73 -
 ...thAntiMatterTuplesDataflowHelperFactory.java |   71 -
 .../am/lsm/rtree/impls/AbstractLSMRTree.java    |  448 ------
 .../am/lsm/rtree/impls/ExternalRTree.java       |  764 ----------
 .../lsm/rtree/impls/ExternalRTreeOpContext.java |  122 --
 .../storage/am/lsm/rtree/impls/LSMRTree.java    |  636 ---------
 .../lsm/rtree/impls/LSMRTreeAbstractCursor.java |  171 ---
 .../rtree/impls/LSMRTreeCursorInitialState.java |  118 --
 .../LSMRTreeDeletedKeysBTreeMergeCursor.java    |   69 -
 .../lsm/rtree/impls/LSMRTreeDiskComponent.java  |   74 -
 .../impls/LSMRTreeDiskComponentFactory.java     |   57 -
 .../am/lsm/rtree/impls/LSMRTreeFileManager.java |  251 ----
 .../lsm/rtree/impls/LSMRTreeFlushOperation.java |  109 --
 .../rtree/impls/LSMRTreeMemoryComponent.java    |   60 -
 .../lsm/rtree/impls/LSMRTreeMergeOperation.java |  128 --
 .../am/lsm/rtree/impls/LSMRTreeOpContext.java   |  172 ---
 .../lsm/rtree/impls/LSMRTreeSearchCursor.java   |  123 --
 .../lsm/rtree/impls/LSMRTreeSortedCursor.java   |  162 ---
 .../impls/LSMRTreeWithAntiMatterTuples.java     |  463 -------
 ...ithAntiMatterTuplesDiskComponentFactory.java |   47 -
 ...LSMRTreeWithAntiMatterTuplesFileManager.java |   30 -
 ...LSMRTreeWithAntiMatterTuplesFlushCursor.java |  167 ---
 ...SMRTreeWithAntiMatterTuplesSearchCursor.java |  285 ----
 .../am/lsm/rtree/impls/RTreeFactory.java        |   43 -
 .../am/lsm/rtree/impls/TreeTupleSorter.java     |  229 ---
 .../rtree/tuples/LSMRTreeCopyTupleWriter.java   |   35 -
 .../tuples/LSMRTreeCopyTupleWriterFactory.java  |   35 -
 .../rtree/tuples/LSMRTreeTupleReference.java    |   47 -
 .../lsm/rtree/tuples/LSMRTreeTupleWriter.java   |   67 -
 .../tuples/LSMRTreeTupleWriterFactory.java      |   38 -
 .../tuples/LSMTypeAwareTupleWriterFactory.java  |   44 -
 .../am/lsm/rtree/utils/LSMRTreeUtils.java       |  286 ----
 .../AbstractLSMRTreeDataflowHelper.java         |   97 ++
 .../dataflow/ExternalRTreeDataflowHelper.java   |  112 ++
 .../ExternalRTreeDataflowHelperFactory.java     |   57 +
 .../rtree/dataflow/LSMRTreeDataflowHelper.java  |   92 ++
 .../dataflow/LSMRTreeDataflowHelperFactory.java |   75 +
 ...RTreeWithAntiMatterTuplesDataflowHelper.java |   73 +
 ...thAntiMatterTuplesDataflowHelperFactory.java |   71 +
 .../am/lsm/rtree/impls/AbstractLSMRTree.java    |  448 ++++++
 .../am/lsm/rtree/impls/ExternalRTree.java       |  764 ++++++++++
 .../lsm/rtree/impls/ExternalRTreeOpContext.java |  122 ++
 .../storage/am/lsm/rtree/impls/LSMRTree.java    |  636 +++++++++
 .../lsm/rtree/impls/LSMRTreeAbstractCursor.java |  171 +++
 .../rtree/impls/LSMRTreeCursorInitialState.java |  118 ++
 .../LSMRTreeDeletedKeysBTreeMergeCursor.java    |   69 +
 .../lsm/rtree/impls/LSMRTreeDiskComponent.java  |   74 +
 .../impls/LSMRTreeDiskComponentFactory.java     |   57 +
 .../am/lsm/rtree/impls/LSMRTreeFileManager.java |  251 ++++
 .../lsm/rtree/impls/LSMRTreeFlushOperation.java |  109 ++
 .../rtree/impls/LSMRTreeMemoryComponent.java    |   60 +
 .../lsm/rtree/impls/LSMRTreeMergeOperation.java |  128 ++
 .../am/lsm/rtree/impls/LSMRTreeOpContext.java   |  172 +++
 .../lsm/rtree/impls/LSMRTreeSearchCursor.java   |  123 ++
 .../lsm/rtree/impls/LSMRTreeSortedCursor.java   |  162 +++
 .../impls/LSMRTreeWithAntiMatterTuples.java     |  463 +++++++
 ...ithAntiMatterTuplesDiskComponentFactory.java |   47 +
 ...LSMRTreeWithAntiMatterTuplesFileManager.java |   30 +
 ...LSMRTreeWithAntiMatterTuplesFlushCursor.java |  167 +++
 ...SMRTreeWithAntiMatterTuplesSearchCursor.java |  285 ++++
 .../am/lsm/rtree/impls/RTreeFactory.java        |   43 +
 .../am/lsm/rtree/impls/TreeTupleSorter.java     |  229 +++
 .../rtree/tuples/LSMRTreeCopyTupleWriter.java   |   35 +
 .../tuples/LSMRTreeCopyTupleWriterFactory.java  |   35 +
 .../rtree/tuples/LSMRTreeTupleReference.java    |   47 +
 .../lsm/rtree/tuples/LSMRTreeTupleWriter.java   |   67 +
 .../tuples/LSMRTreeTupleWriterFactory.java      |   38 +
 .../tuples/LSMTypeAwareTupleWriterFactory.java  |   44 +
 .../am/lsm/rtree/utils/LSMRTreeUtils.java       |  286 ++++
 ...IGenericPrimitiveSerializerDeserializer.java |   23 -
 .../storage/am/rtree/api/IRTreeFrame.java       |   35 -
 .../am/rtree/api/IRTreeInteriorFrame.java       |   44 -
 .../storage/am/rtree/api/IRTreeLeafFrame.java   |   30 -
 .../storage/am/rtree/api/IRTreePolicy.java      |   34 -
 .../am/rtree/dataflow/RTreeDataflowHelper.java  |   49 -
 .../dataflow/RTreeDataflowHelperFactory.java    |   45 -
 .../dataflow/RTreeSearchOperatorDescriptor.java |   66 -
 .../RTreeSearchOperatorNodePushable.java        |   67 -
 .../am/rtree/frames/RStarTreePolicy.java        |  355 -----
 .../am/rtree/frames/RTreeComputationUtils.java  |  139 --
 .../storage/am/rtree/frames/RTreeNSMFrame.java  |  163 ---
 .../am/rtree/frames/RTreeNSMInteriorFrame.java  |  298 ----
 .../frames/RTreeNSMInteriorFrameFactory.java    |   54 -
 .../am/rtree/frames/RTreeNSMLeafFrame.java      |  122 --
 .../rtree/frames/RTreeNSMLeafFrameFactory.java  |   54 -
 .../storage/am/rtree/frames/RTreePolicy.java    |  233 ----
 .../am/rtree/frames/RTreePolicyType.java        |   20 -
 .../DoublePrimitiveValueProviderFactory.java    |   38 -
 .../storage/am/rtree/impls/EntriesOrder.java    |   20 -
 .../FloatPrimitiveValueProviderFactory.java     |   38 -
 .../IntegerPrimitiveValueProviderFactory.java   |   38 -
 .../storage/am/rtree/impls/PathList.java        |  117 --
 .../hyracks/storage/am/rtree/impls/RTree.java   | 1001 --------------
 .../am/rtree/impls/RTreeCursorInitialState.java |   74 -
 .../storage/am/rtree/impls/RTreeOpContext.java  |  116 --
 .../am/rtree/impls/RTreeSearchCursor.java       |  260 ----
 .../storage/am/rtree/impls/RTreeSplitKey.java   |  154 ---
 .../storage/am/rtree/impls/Rectangle.java       |  138 --
 .../storage/am/rtree/impls/SearchPredicate.java |   56 -
 .../storage/am/rtree/impls/TupleEntry.java      |   52 -
 .../am/rtree/impls/TupleEntryArrayList.java     |   79 --
 .../am/rtree/impls/UnorderedSlotManager.java    |  124 --
 .../linearize/HilbertDoubleComparator.java      |  194 ---
 .../HilbertDoubleComparatorFactory.java         |   37 -
 .../rtree/linearize/ZCurveDoubleComparator.java |  150 --
 .../ZCurveDoubleComparatorFactory.java          |   37 -
 .../am/rtree/linearize/ZCurveIntComparator.java |  144 --
 .../linearize/ZCurveIntComparatorFactory.java   |   37 -
 .../rtree/tuples/RTreeTypeAwareTupleWriter.java |   57 -
 .../RTreeTypeAwareTupleWriterFactory.java       |   35 -
 .../storage/am/rtree/util/RTreeUtils.java       |   79 --
 ...IGenericPrimitiveSerializerDeserializer.java |   23 +
 .../storage/am/rtree/api/IRTreeFrame.java       |   35 +
 .../am/rtree/api/IRTreeInteriorFrame.java       |   44 +
 .../storage/am/rtree/api/IRTreeLeafFrame.java   |   30 +
 .../storage/am/rtree/api/IRTreePolicy.java      |   34 +
 .../am/rtree/dataflow/RTreeDataflowHelper.java  |   49 +
 .../dataflow/RTreeDataflowHelperFactory.java    |   45 +
 .../dataflow/RTreeSearchOperatorDescriptor.java |   66 +
 .../RTreeSearchOperatorNodePushable.java        |   67 +
 .../am/rtree/frames/RStarTreePolicy.java        |  355 +++++
 .../am/rtree/frames/RTreeComputationUtils.java  |  139 ++
 .../storage/am/rtree/frames/RTreeNSMFrame.java  |  163 +++
 .../am/rtree/frames/RTreeNSMInteriorFrame.java  |  298 ++++
 .../frames/RTreeNSMInteriorFrameFactory.java    |   54 +
 .../am/rtree/frames/RTreeNSMLeafFrame.java      |  122 ++
 .../rtree/frames/RTreeNSMLeafFrameFactory.java  |   54 +
 .../storage/am/rtree/frames/RTreePolicy.java    |  233 ++++
 .../am/rtree/frames/RTreePolicyType.java        |   20 +
 .../DoublePrimitiveValueProviderFactory.java    |   38 +
 .../storage/am/rtree/impls/EntriesOrder.java    |   20 +
 .../FloatPrimitiveValueProviderFactory.java     |   38 +
 .../IntegerPrimitiveValueProviderFactory.java   |   38 +
 .../storage/am/rtree/impls/PathList.java        |  117 ++
 .../hyracks/storage/am/rtree/impls/RTree.java   | 1001 ++++++++++++++
 .../am/rtree/impls/RTreeCursorInitialState.java |   74 +
 .../storage/am/rtree/impls/RTreeOpContext.java  |  116 ++
 .../am/rtree/impls/RTreeSearchCursor.java       |  260 ++++
 .../storage/am/rtree/impls/RTreeSplitKey.java   |  154 +++
 .../storage/am/rtree/impls/Rectangle.java       |  138 ++
 .../storage/am/rtree/impls/SearchPredicate.java |   56 +
 .../storage/am/rtree/impls/TupleEntry.java      |   52 +
 .../am/rtree/impls/TupleEntryArrayList.java     |   79 ++
 .../am/rtree/impls/UnorderedSlotManager.java    |  124 ++
 .../linearize/HilbertDoubleComparator.java      |  194 +++
 .../HilbertDoubleComparatorFactory.java         |   37 +
 .../rtree/linearize/ZCurveDoubleComparator.java |  150 ++
 .../ZCurveDoubleComparatorFactory.java          |   37 +
 .../am/rtree/linearize/ZCurveIntComparator.java |  144 ++
 .../linearize/ZCurveIntComparatorFactory.java   |   37 +
 .../rtree/tuples/RTreeTypeAwareTupleWriter.java |   57 +
 .../RTreeTypeAwareTupleWriterFactory.java       |   35 +
 .../storage/am/rtree/util/RTreeUtils.java       |   79 ++
 .../common/IStorageManagerInterface.java        |   33 -
 .../storage/common/buffercache/BufferCache.java |  834 -----------
 .../storage/common/buffercache/CachedPage.java  |  110 --
 .../ClockPageReplacementStrategy.java           |  139 --
 .../common/buffercache/DebugBufferCache.java    |  206 ---
 .../buffercache/DelayPageCleanerPolicy.java     |   44 -
 .../buffercache/DirectBufferAllocator.java      |   28 -
 .../common/buffercache/HeapBufferAllocator.java |   28 -
 .../common/buffercache/IBufferCache.java        |   60 -
 .../buffercache/IBufferCacheInternal.java       |   21 -
 .../buffercache/ICacheMemoryAllocator.java      |   21 -
 .../storage/common/buffercache/ICachedPage.java |   29 -
 .../common/buffercache/ICachedPageInternal.java |   23 -
 .../common/buffercache/IPageCleanerPolicy.java  |   51 -
 .../buffercache/IPageReplacementStrategy.java   |   33 -
 .../storage/common/file/BufferedFileHandle.java |   75 -
 .../storage/common/file/IFileMapManager.java    |   49 -
 .../storage/common/file/IFileMapProvider.java   |   61 -
 .../common/file/ILocalResourceFactory.java      |   19 -
 .../file/ILocalResourceFactoryProvider.java     |   21 -
 .../common/file/ILocalResourceRepository.java   |   34 -
 .../file/ILocalResourceRepositoryFactory.java   |   21 -
 .../storage/common/file/LocalResource.java      |   62 -
 .../file/NoOpLocalResourceFactoryProvider.java  |   27 -
 .../storage/common/file/ResourceIdFactory.java  |   33 -
 .../common/file/ResourceIdFactoryProvider.java  |   38 -
 .../common/file/TransientFileMapManager.java    |   82 --
 .../file/TransientLocalResourceFactory.java     |   23 -
 .../TransientLocalResourceFactoryProvider.java  |   25 -
 .../file/TransientLocalResourceRepository.java  |   78 --
 ...TransientLocalResourceRepositoryFactory.java |   25 -
 .../hyracks/storage/common/sync/LatchType.java  |   20 -
 .../common/IStorageManagerInterface.java        |   33 +
 .../storage/common/buffercache/BufferCache.java |  834 +++++++++++
 .../storage/common/buffercache/CachedPage.java  |  110 ++
 .../ClockPageReplacementStrategy.java           |  139 ++
 .../common/buffercache/DebugBufferCache.java    |  206 +++
 .../buffercache/DelayPageCleanerPolicy.java     |   44 +
 .../buffercache/DirectBufferAllocator.java      |   28 +
 .../common/buffercache/HeapBufferAllocator.java |   28 +
 .../common/buffercache/IBufferCache.java        |   60 +
 .../buffercache/IBufferCacheInternal.java       |   21 +
 .../buffercache/ICacheMemoryAllocator.java      |   21 +
 .../storage/common/buffercache/ICachedPage.java |   29 +
 .../common/buffercache/ICachedPageInternal.java |   23 +
 .../common/buffercache/IPageCleanerPolicy.java  |   51 +
 .../buffercache/IPageReplacementStrategy.java   |   33 +
 .../storage/common/file/BufferedFileHandle.java |   75 +
 .../storage/common/file/IFileMapManager.java    |   49 +
 .../storage/common/file/IFileMapProvider.java   |   61 +
 .../common/file/ILocalResourceFactory.java      |   19 +
 .../file/ILocalResourceFactoryProvider.java     |   21 +
 .../common/file/ILocalResourceRepository.java   |   34 +
 .../file/ILocalResourceRepositoryFactory.java   |   21 +
 .../storage/common/file/LocalResource.java      |   62 +
 .../file/NoOpLocalResourceFactoryProvider.java  |   27 +
 .../storage/common/file/ResourceIdFactory.java  |   33 +
 .../common/file/ResourceIdFactoryProvider.java  |   38 +
 .../common/file/TransientFileMapManager.java    |   82 ++
 .../file/TransientLocalResourceFactory.java     |   23 +
 .../TransientLocalResourceFactoryProvider.java  |   25 +
 .../file/TransientLocalResourceRepository.java  |   78 ++
 ...TransientLocalResourceRepositoryFactory.java |   25 +
 .../hyracks/storage/common/sync/LatchType.java  |   20 +
 ...stractModificationOperationCallbackTest.java |   99 --
 .../am/btree/AbstractOperationCallbackTest.java |   54 -
 .../AbstractSearchOperationCallbackTest.java    |  249 ----
 .../am/btree/OrderedIndexBulkLoadTest.java      |   70 -
 .../am/btree/OrderedIndexDeleteTest.java        |   76 -
 .../am/btree/OrderedIndexExamplesTest.java      |  834 -----------
 .../am/btree/OrderedIndexInsertTest.java        |   75 -
 .../am/btree/OrderedIndexMultiThreadTest.java   |  128 --
 .../am/btree/OrderedIndexSortedInsertTest.java  |   75 -
 .../am/btree/OrderedIndexTestContext.java       |   48 -
 .../am/btree/OrderedIndexTestDriver.java        |  179 ---
 .../storage/am/btree/OrderedIndexTestUtils.java |  517 -------
 .../am/btree/OrderedIndexUpdateTest.java        |   76 -
 .../am/btree/OrderedIndexUpsertTest.java        |   74 -
 .../am/common/AbstractIndexLifecycleTest.java   |  123 --
 .../am/common/AbstractIndexTestWorker.java      |   73 -
 .../hyracks/storage/am/common/CheckTuple.java   |  106 --
 .../storage/am/common/IIndexTestContext.java    |   51 -
 .../am/common/IIndexTestWorkerFactory.java      |   25 -
 .../am/common/ITreeIndexTestContext.java        |   51 -
 .../storage/am/common/ITreeIndexTestWorker.java |   25 -
 .../am/common/IndexMultiThreadTestDriver.java   |   91 --
 .../storage/am/common/IndexTestContext.java     |   82 --
 .../am/common/TestOperationCallback.java        |   65 -
 .../am/common/TestOperationSelector.java        |   67 -
 .../storage/am/common/TestWorkloadConf.java     |   38 -
 .../storage/am/common/TreeIndexTestUtils.java   |  298 ----
 .../am/config/AccessMethodTestsConfig.java      |  206 ---
 .../am/rtree/AbstractRTreeBulkLoadTest.java     |   63 -
 .../am/rtree/AbstractRTreeDeleteTest.java       |   71 -
 .../am/rtree/AbstractRTreeExamplesTest.java     |  917 ------------
 .../am/rtree/AbstractRTreeInsertTest.java       |   68 -
 .../am/rtree/AbstractRTreeMultiThreadTest.java  |  277 ----
 .../am/rtree/AbstractRTreeTestContext.java      |   39 -
 .../am/rtree/AbstractRTreeTestDriver.java       |  209 ---
 .../storage/am/rtree/RTreeCheckTuple.java       |   56 -
 .../storage/am/rtree/RTreeTestUtils.java        |  254 ----
 .../hyracks/test/support/CounterContext.java    |   48 -
 .../TestIndexLifecycleManagerProvider.java      |   30 -
 .../hyracks/test/support/TestJobletContext.java |  125 --
 .../test/support/TestNCApplicationContext.java  |  127 --
 .../hyracks/test/support/TestRootContext.java   |   48 -
 .../TestStorageManagerComponentHolder.java      |  132 --
 .../support/TestStorageManagerInterface.java    |   47 -
 .../hyracks/test/support/TestTaskContext.java   |  132 --
 .../uci/ics/hyracks/test/support/TestUtils.java |   40 -
 .../support/TestVirtualBufferCacheProvider.java |   47 -
 ...stractModificationOperationCallbackTest.java |   99 ++
 .../am/btree/AbstractOperationCallbackTest.java |   54 +
 .../AbstractSearchOperationCallbackTest.java    |  249 ++++
 .../am/btree/OrderedIndexBulkLoadTest.java      |   70 +
 .../am/btree/OrderedIndexDeleteTest.java        |   76 +
 .../am/btree/OrderedIndexExamplesTest.java      |  834 +++++++++++
 .../am/btree/OrderedIndexInsertTest.java        |   75 +
 .../am/btree/OrderedIndexMultiThreadTest.java   |  128 ++
 .../am/btree/OrderedIndexSortedInsertTest.java  |   75 +
 .../am/btree/OrderedIndexTestContext.java       |   48 +
 .../am/btree/OrderedIndexTestDriver.java        |  179 +++
 .../storage/am/btree/OrderedIndexTestUtils.java |  517 +++++++
 .../am/btree/OrderedIndexUpdateTest.java        |   76 +
 .../am/btree/OrderedIndexUpsertTest.java        |   74 +
 .../am/common/AbstractIndexLifecycleTest.java   |  123 ++
 .../am/common/AbstractIndexTestWorker.java      |   73 +
 .../hyracks/storage/am/common/CheckTuple.java   |  106 ++
 .../storage/am/common/IIndexTestContext.java    |   51 +
 .../am/common/IIndexTestWorkerFactory.java      |   25 +
 .../am/common/ITreeIndexTestContext.java        |   51 +
 .../storage/am/common/ITreeIndexTestWorker.java |   25 +
 .../am/common/IndexMultiThreadTestDriver.java   |   91 ++
 .../storage/am/common/IndexTestContext.java     |   82 ++
 .../am/common/TestOperationCallback.java        |   65 +
 .../am/common/TestOperationSelector.java        |   67 +
 .../storage/am/common/TestWorkloadConf.java     |   38 +
 .../storage/am/common/TreeIndexTestUtils.java   |  298 ++++
 .../am/config/AccessMethodTestsConfig.java      |  206 +++
 .../am/rtree/AbstractRTreeBulkLoadTest.java     |   63 +
 .../am/rtree/AbstractRTreeDeleteTest.java       |   71 +
 .../am/rtree/AbstractRTreeExamplesTest.java     |  917 ++++++++++++
 .../am/rtree/AbstractRTreeInsertTest.java       |   68 +
 .../am/rtree/AbstractRTreeMultiThreadTest.java  |  277 ++++
 .../am/rtree/AbstractRTreeTestContext.java      |   39 +
 .../am/rtree/AbstractRTreeTestDriver.java       |  209 +++
 .../storage/am/rtree/RTreeCheckTuple.java       |   56 +
 .../storage/am/rtree/RTreeTestUtils.java        |  254 ++++
 .../hyracks/test/support/CounterContext.java    |   48 +
 .../TestIndexLifecycleManagerProvider.java      |   30 +
 .../hyracks/test/support/TestJobletContext.java |  125 ++
 .../test/support/TestNCApplicationContext.java  |  127 ++
 .../hyracks/test/support/TestRootContext.java   |   48 +
 .../TestStorageManagerComponentHolder.java      |  132 ++
 .../support/TestStorageManagerInterface.java    |   47 +
 .../hyracks/test/support/TestTaskContext.java   |  132 ++
 .../apache/hyracks/test/support/TestUtils.java  |   40 +
 .../support/TestVirtualBufferCacheProvider.java |   47 +
 .../storage/am/bloomfilter/BloomFilterTest.java |  167 ---
 .../MurmurHashForITupleReferenceTest.java       |  296 ----
 .../util/AbstractBloomFilterTest.java           |   58 -
 .../util/BloomFilterTestHarness.java            |  120 --
 .../storage/am/bloomfilter/BloomFilterTest.java |  167 +++
 .../MurmurHashForITupleReferenceTest.java       |  296 ++++
 .../util/AbstractBloomFilterTest.java           |   58 +
 .../util/BloomFilterTestHarness.java            |  120 ++
 .../storage/am/btree/BTreeBulkLoadTest.java     |   59 -
 .../storage/am/btree/BTreeDeleteTest.java       |   59 -
 .../storage/am/btree/BTreeExamplesTest.java     |   50 -
 .../storage/am/btree/BTreeInsertTest.java       |   68 -
 .../storage/am/btree/BTreeLifecycleTest.java    |   89 --
 .../BTreeModificationOperationCallbackTest.java |   49 -
 .../storage/am/btree/BTreeSearchCursorTest.java |  429 ------
 .../btree/BTreeSearchOperationCallbackTest.java |   49 -
 .../storage/am/btree/BTreeSortedInsertTest.java |   68 -
 .../storage/am/btree/BTreeStatsTest.java        |  176 ---
 .../storage/am/btree/BTreeUpdateSearchTest.java |  170 ---
 .../storage/am/btree/BTreeUpdateTest.java       |   59 -
 .../storage/am/btree/BTreeUpsertTest.java       |   68 -
 .../storage/am/btree/FieldPrefixNSMTest.java    |  233 ----
 .../storage/am/btree/StorageManagerTest.java    |  267 ----
 .../btree/multithread/BTreeMultiThreadTest.java |   95 --
 .../am/btree/multithread/BTreeTestWorker.java   |  134 --
 .../multithread/BTreeTestWorkerFactory.java     |   31 -
 .../am/btree/util/AbstractBTreeTest.java        |   47 -
 .../storage/am/btree/util/BTreeTestContext.java |   59 -
 .../storage/am/btree/util/BTreeTestHarness.java |  123 --
 .../storage/am/btree/BTreeBulkLoadTest.java     |   59 +
 .../storage/am/btree/BTreeDeleteTest.java       |   59 +
 .../storage/am/btree/BTreeExamplesTest.java     |   50 +
 .../storage/am/btree/BTreeInsertTest.java       |   68 +
 .../storage/am/btree/BTreeLifecycleTest.java    |   89 ++
 .../BTreeModificationOperationCallbackTest.java |   49 +
 .../storage/am/btree/BTreeSearchCursorTest.java |  429 ++++++
 .../btree/BTreeSearchOperationCallbackTest.java |   49 +
 .../storage/am/btree/BTreeSortedInsertTest.java |   68 +
 .../storage/am/btree/BTreeStatsTest.java        |  176 +++
 .../storage/am/btree/BTreeUpdateSearchTest.java |  170 +++
 .../storage/am/btree/BTreeUpdateTest.java       |   59 +
 .../storage/am/btree/BTreeUpsertTest.java       |   68 +
 .../storage/am/btree/FieldPrefixNSMTest.java    |  233 ++++
 .../storage/am/btree/StorageManagerTest.java    |  267 ++++
 .../btree/multithread/BTreeMultiThreadTest.java |   95 ++
 .../am/btree/multithread/BTreeTestWorker.java   |  134 ++
 .../multithread/BTreeTestWorkerFactory.java     |   31 +
 .../am/btree/util/AbstractBTreeTest.java        |   47 +
 .../storage/am/btree/util/BTreeTestContext.java |   59 +
 .../storage/am/btree/util/BTreeTestHarness.java |  123 ++
 .../am/lsm/btree/LSMBTreeBulkLoadTest.java      |   64 -
 .../am/lsm/btree/LSMBTreeDeleteTest.java        |   64 -
 .../am/lsm/btree/LSMBTreeExamplesTest.java      |  160 ---
 .../am/lsm/btree/LSMBTreeInsertTest.java        |   64 -
 .../am/lsm/btree/LSMBTreeLifecycleTest.java     |   89 --
 .../storage/am/lsm/btree/LSMBTreeMergeTest.java |   63 -
 .../am/lsm/btree/LSMBTreeMergeTestDriver.java   |   93 --
 ...MBTreeModificationOperationCallbackTest.java |  108 --
 .../am/lsm/btree/LSMBTreeMultiBulkLoadTest.java |   65 -
 .../LSMBTreeSearchOperationCallbackTest.java    |  287 ----
 .../am/lsm/btree/LSMBTreeUpdateTest.java        |   64 -
 .../multithread/LSMBTreeMultiThreadTest.java    |  115 --
 .../btree/multithread/LSMBTreeTestWorker.java   |  116 --
 .../multithread/LSMBTreeTestWorkerFactory.java  |   31 -
 .../am/lsm/btree/perf/BTreeBulkLoadRunner.java  |   52 -
 .../am/lsm/btree/perf/BTreePageSizePerf.java    |   85 --
 .../storage/am/lsm/btree/perf/BTreeRunner.java  |   48 -
 .../btree/perf/ConcurrentSkipListRunner.java    |  145 --
 .../am/lsm/btree/perf/IExperimentRunner.java    |   30 -
 .../am/lsm/btree/perf/InMemoryBTreeRunner.java  |  141 --
 .../am/lsm/btree/perf/InMemorySortRunner.java   |  160 ---
 .../am/lsm/btree/perf/LSMTreeRunner.java        |  195 ---
 .../am/lsm/btree/perf/PerfExperiment.java       |  103 --
 .../am/lsm/btree/tuples/LSMBTreeTuplesTest.java |  171 ---
 .../am/lsm/btree/util/LSMBTreeTestContext.java  |   83 --
 .../am/lsm/btree/util/LSMBTreeTestHarness.java  |  212 ---
 .../am/lsm/btree/LSMBTreeBulkLoadTest.java      |   64 +
 .../am/lsm/btree/LSMBTreeDeleteTest.java        |   64 +
 .../am/lsm/btree/LSMBTreeExamplesTest.java      |  160 +++
 .../am/lsm/btree/LSMBTreeInsertTest.java        |   64 +
 .../am/lsm/btree/LSMBTreeLifecycleTest.java     |   89 ++
 .../storage/am/lsm/btree/LSMBTreeMergeTest.java |   63 +
 .../am/lsm/btree/LSMBTreeMergeTestDriver.java   |   93 ++
 ...MBTreeModificationOperationCallbackTest.java |  108 ++
 .../am/lsm/btree/LSMBTreeMultiBulkLoadTest.java |   65 +
 .../LSMBTreeSearchOperationCallbackTest.java    |  287 ++++
 .../am/lsm/btree/LSMBTreeUpdateTest.java        |   64 +
 .../multithread/LSMBTreeMultiThreadTest.java    |  115 ++
 .../btree/multithread/LSMBTreeTestWorker.java   |  116 ++
 .../multithread/LSMBTreeTestWorkerFactory.java  |   31 +
 .../am/lsm/btree/perf/BTreeBulkLoadRunner.java  |   52 +
 .../am/lsm/btree/perf/BTreePageSizePerf.java    |   85 ++
 .../storage/am/lsm/btree/perf/BTreeRunner.java  |   48 +
 .../btree/perf/ConcurrentSkipListRunner.java    |  145 ++
 .../am/lsm/btree/perf/IExperimentRunner.java    |   30 +
 .../am/lsm/btree/perf/InMemoryBTreeRunner.java  |  141 ++
 .../am/lsm/btree/perf/InMemorySortRunner.java   |  160 +++
 .../am/lsm/btree/perf/LSMTreeRunner.java        |  195 +++
 .../am/lsm/btree/perf/PerfExperiment.java       |  103 ++
 .../am/lsm/btree/tuples/LSMBTreeTuplesTest.java |  171 +++
 .../am/lsm/btree/util/LSMBTreeTestContext.java  |   83 ++
 .../am/lsm/btree/util/LSMBTreeTestHarness.java  |  212 +++
 .../am/lsm/common/DummyLSMIndexFileManager.java |   48 -
 .../storage/am/lsm/common/DummyTreeFactory.java |   34 -
 .../am/lsm/common/LSMIndexFileManagerTest.java  |  257 ----
 .../am/lsm/common/VirtualBufferCacheTest.java   |  132 --
 .../lsm/common/VirtualFreePageManagerTest.java  |   58 -
 .../am/lsm/common/DummyLSMIndexFileManager.java |   48 +
 .../storage/am/lsm/common/DummyTreeFactory.java |   34 +
 .../am/lsm/common/LSMIndexFileManagerTest.java  |  257 ++++
 .../am/lsm/common/VirtualBufferCacheTest.java   |  132 ++
 .../lsm/common/VirtualFreePageManagerTest.java  |   58 +
 .../LSMInvertedIndexBulkLoadTest.java           |   26 -
 .../LSMInvertedIndexDeleteTest.java             |   26 -
 .../LSMInvertedIndexInsertTest.java             |   26 -
 .../LSMInvertedIndexMergeTest.java              |   67 -
 .../LSMInvertedIndexSearchTest.java             |   26 -
 ...PartitionedLSMInvertedIndexBulkLoadTest.java |   26 -
 .../PartitionedLSMInvertedIndexDeleteTest.java  |   26 -
 .../PartitionedLSMInvertedIndexInsertTest.java  |   26 -
 .../PartitionedLSMInvertedIndexMergeTest.java   |   67 -
 .../PartitionedLSMInvertedIndexSearchTest.java  |   26 -
 .../common/AbstractInvertedIndexDeleteTest.java |  100 --
 .../common/AbstractInvertedIndexLoadTest.java   |   87 --
 .../common/AbstractInvertedIndexSearchTest.java |  128 --
 .../common/AbstractInvertedIndexTest.java       |  101 --
 .../common/LSMInvertedIndexTestHarness.java     |  218 ---
 .../InMemoryInvertedIndexDeleteTest.java        |   26 -
 .../InMemoryInvertedIndexInsertTest.java        |   26 -
 .../InMemoryInvertedIndexSearchTest.java        |   26 -
 ...titionedInMemoryInvertedIndexDeleteTest.java |   26 -
 ...titionedInMemoryInvertedIndexInsertTest.java |   26 -
 ...titionedInMemoryInvertedIndexSearchTest.java |   26 -
 .../LSMInvertedIndexMultiThreadTest.java        |  162 ---
 .../LSMInvertedIndexMultiThreadTestDriver.java  |   41 -
 .../multithread/LSMInvertedIndexTestWorker.java |  129 --
 .../LSMInvertedIndexWorkerFactory.java          |   31 -
 ...titionedLSMInvertedIndexMultiThreadTest.java |   25 -
 .../ondisk/FixedSizeFrameTupleTest.java         |   75 -
 .../ondisk/OnDiskInvertedIndexBulkLoadTest.java |   26 -
 .../OnDiskInvertedIndexLifecycleTest.java       |   93 --
 .../ondisk/OnDiskInvertedIndexSearchTest.java   |   26 -
 ...titionedOnDiskInvertedIndexBulkLoadTest.java |   26 -
 ...artitionedOnDiskInvertedIndexSearchTest.java |   26 -
 .../tokenizers/NGramTokenizerTest.java          |  228 ---
 .../tokenizers/WordTokenizerTest.java           |  210 ---
 .../util/LSMInvertedIndexTestContext.java       |  259 ----
 .../util/LSMInvertedIndexTestUtils.java         |  567 --------
 .../LSMInvertedIndexBulkLoadTest.java           |   26 +
 .../LSMInvertedIndexDeleteTest.java             |   26 +
 .../LSMInvertedIndexInsertTest.java             |   26 +
 .../LSMInvertedIndexMergeTest.java              |   67 +
 .../LSMInvertedIndexSearchTest.java             |   26 +
 ...PartitionedLSMInvertedIndexBulkLoadTest.java |   26 +
 .../PartitionedLSMInvertedIndexDeleteTest.java  |   26 +
 .../PartitionedLSMInvertedIndexInsertTest.java  |   26 +
 .../PartitionedLSMInvertedIndexMergeTest.java   |   67 +
 .../PartitionedLSMInvertedIndexSearchTest.java  |   26 +
 .../common/AbstractInvertedIndexDeleteTest.java |  100 ++
 .../common/AbstractInvertedIndexLoadTest.java   |   87 ++
 .../common/AbstractInvertedIndexSearchTest.java |  128 ++
 .../common/AbstractInvertedIndexTest.java       |  101 ++
 .../common/LSMInvertedIndexTestHarness.java     |  218 +++
 .../InMemoryInvertedIndexDeleteTest.java        |   26 +
 .../InMemoryInvertedIndexInsertTest.java        |   26 +
 .../InMemoryInvertedIndexSearchTest.java        |   26 +
 ...titionedInMemoryInvertedIndexDeleteTest.java |   26 +
 ...titionedInMemoryInvertedIndexInsertTest.java |   26 +
 ...titionedInMemoryInvertedIndexSearchTest.java |   26 +
 .../LSMInvertedIndexMultiThreadTest.java        |  162 +++
 .../LSMInvertedIndexMultiThreadTestDriver.java  |   41 +
 .../multithread/LSMInvertedIndexTestWorker.java |  129 ++
 .../LSMInvertedIndexWorkerFactory.java          |   31 +
 ...titionedLSMInvertedIndexMultiThreadTest.java |   25 +
 .../ondisk/FixedSizeFrameTupleTest.java         |   75 +
 .../ondisk/OnDiskInvertedIndexBulkLoadTest.java |   26 +
 .../OnDiskInvertedIndexLifecycleTest.java       |   93 ++
 .../ondisk/OnDiskInvertedIndexSearchTest.java   |   26 +
 ...titionedOnDiskInvertedIndexBulkLoadTest.java |   26 +
 ...artitionedOnDiskInvertedIndexSearchTest.java |   26 +
 .../tokenizers/NGramTokenizerTest.java          |  228 +++
 .../tokenizers/WordTokenizerTest.java           |  210 +++
 .../util/LSMInvertedIndexTestContext.java       |  259 ++++
 .../util/LSMInvertedIndexTestUtils.java         |  567 ++++++++
 .../lsm/rtree/AbstractLSMRTreeExamplesTest.java |  168 ---
 .../am/lsm/rtree/LSMRTreeBulkLoadTest.java      |   67 -
 .../am/lsm/rtree/LSMRTreeDeleteTest.java        |   67 -
 .../am/lsm/rtree/LSMRTreeExamplesTest.java      |   64 -
 .../am/lsm/rtree/LSMRTreeInsertTest.java        |   67 -
 .../am/lsm/rtree/LSMRTreeLifecycleTest.java     |   98 --
 .../storage/am/lsm/rtree/LSMRTreeMergeTest.java |   66 -
 .../am/lsm/rtree/LSMRTreeMergeTestDriver.java   |   91 --
 ...SMRTreeWithAntiMatterTuplesBulkLoadTest.java |   67 -
 .../LSMRTreeWithAntiMatterTuplesDeleteTest.java |   67 -
 ...SMRTreeWithAntiMatterTuplesExamplesTest.java |   64 -
 .../LSMRTreeWithAntiMatterTuplesInsertTest.java |   67 -
 ...MRTreeWithAntiMatterTuplesLifecycleTest.java |   98 --
 .../LSMRTreeWithAntiMatterTuplesMergeTest.java  |   66 -
 .../multithread/AbstractLSMRTreeTestWorker.java |   86 --
 .../multithread/LSMRTreeMultiThreadTest.java    |  124 --
 .../rtree/multithread/LSMRTreeTestWorker.java   |  124 --
 .../multithread/LSMRTreeTestWorkerFactory.java  |   31 -
 ...TreeWithAntiMatterTuplesMultiThreadTest.java |  125 --
 .../LSMRTreeWithAntiMatterTuplesTestWorker.java |   73 -
 ...eeWithAntiMatterTuplesTestWorkerFactory.java |   31 -
 .../am/lsm/rtree/util/LSMRTreeTestContext.java  |   95 --
 .../am/lsm/rtree/util/LSMRTreeTestHarness.java  |  212 ---
 ...LSMRTreeWithAntiMatterTuplesTestContext.java |   92 --
 .../lsm/rtree/AbstractLSMRTreeExamplesTest.java |  168 +++
 .../am/lsm/rtree/LSMRTreeBulkLoadTest.java      |   67 +
 .../am/lsm/rtree/LSMRTreeDeleteTest.java        |   67 +
 .../am/lsm/rtree/LSMRTreeExamplesTest.java      |   64 +
 .../am/lsm/rtree/LSMRTreeInsertTest.java        |   67 +
 .../am/lsm/rtree/LSMRTreeLifecycleTest.java     |   98 ++
 .../storage/am/lsm/rtree/LSMRTreeMergeTest.java |   66 +
 .../am/lsm/rtree/LSMRTreeMergeTestDriver.java   |   91 ++
 ...SMRTreeWithAntiMatterTuplesBulkLoadTest.java |   67 +
 .../LSMRTreeWithAntiMatterTuplesDeleteTest.java |   67 +
 ...SMRTreeWithAntiMatterTuplesExamplesTest.java |   64 +
 .../LSMRTreeWithAntiMatterTuplesInsertTest.java |   67 +
 ...MRTreeWithAntiMatterTuplesLifecycleTest.java |   98 ++
 .../LSMRTreeWithAntiMatterTuplesMergeTest.java  |   66 +
 .../multithread/AbstractLSMRTreeTestWorker.java |   86 ++
 .../multithread/LSMRTreeMultiThreadTest.java    |  124 ++
 .../rtree/multithread/LSMRTreeTestWorker.java   |  124 ++
 .../multithread/LSMRTreeTestWorkerFactory.java  |   31 +
 ...TreeWithAntiMatterTuplesMultiThreadTest.java |  125 ++
 .../LSMRTreeWithAntiMatterTuplesTestWorker.java |   73 +
 ...eeWithAntiMatterTuplesTestWorkerFactory.java |   31 +
 .../am/lsm/rtree/util/LSMRTreeTestContext.java  |   95 ++
 .../am/lsm/rtree/util/LSMRTreeTestHarness.java  |  212 +++
 ...LSMRTreeWithAntiMatterTuplesTestContext.java |   92 ++
 .../storage/am/rtree/RTreeBulkLoadTest.java     |   62 -
 .../storage/am/rtree/RTreeDeleteTest.java       |   62 -
 .../storage/am/rtree/RTreeExamplesTest.java     |   58 -
 .../storage/am/rtree/RTreeInsertTest.java       |   62 -
 .../storage/am/rtree/RTreeLifecycleTest.java    |   97 --
 .../storage/am/rtree/RTreeSearchCursorTest.java |  178 ---
 .../rtree/multithread/RTreeMultiThreadTest.java |  102 --
 .../am/rtree/multithread/RTreeTestWorker.java   |  116 --
 .../multithread/RTreeTestWorkerFactory.java     |   31 -
 .../am/rtree/utils/AbstractRTreeTest.java       |   46 -
 .../am/rtree/utils/RTreeTestContext.java        |   63 -
 .../am/rtree/utils/RTreeTestHarness.java        |  122 --
 .../storage/am/rtree/RTreeBulkLoadTest.java     |   62 +
 .../storage/am/rtree/RTreeDeleteTest.java       |   62 +
 .../storage/am/rtree/RTreeExamplesTest.java     |   58 +
 .../storage/am/rtree/RTreeInsertTest.java       |   62 +
 .../storage/am/rtree/RTreeLifecycleTest.java    |   97 ++
 .../storage/am/rtree/RTreeSearchCursorTest.java |  178 +++
 .../rtree/multithread/RTreeMultiThreadTest.java |  102 ++
 .../am/rtree/multithread/RTreeTestWorker.java   |  116 ++
 .../multithread/RTreeTestWorkerFactory.java     |   31 +
 .../am/rtree/utils/AbstractRTreeTest.java       |   46 +
 .../am/rtree/utils/RTreeTestContext.java        |   63 +
 .../am/rtree/utils/RTreeTestHarness.java        |  122 ++
 .../common/BufferCacheRegressionTest.java       |  149 --
 .../hyracks/storage/common/BufferCacheTest.java |  421 ------
 .../common/BufferCacheRegressionTest.java       |  149 ++
 .../hyracks/storage/common/BufferCacheTest.java |  421 ++++++
 .../yarn/am/HyracksYarnApplicationMaster.java   |  251 ----
 .../yarn/am/manifest/AbstractProcess.java       |   37 -
 .../yarn/am/manifest/ClusterController.java     |   18 -
 .../am/manifest/ContainerSpecification.java     |   41 -
 .../yarn/am/manifest/HyracksCluster.java        |   54 -
 .../yarn/am/manifest/ManifestParser.java        |   47 -
 .../yarn/am/manifest/NodeController.java        |   27 -
 .../yarn/am/HyracksYarnApplicationMaster.java   |  251 ++++
 .../yarn/am/manifest/AbstractProcess.java       |   37 +
 .../yarn/am/manifest/ClusterController.java     |   18 +
 .../am/manifest/ContainerSpecification.java     |   41 +
 .../yarn/am/manifest/HyracksCluster.java        |   54 +
 .../yarn/am/manifest/ManifestParser.java        |   47 +
 .../yarn/am/manifest/NodeController.java        |   27 +
 .../yarn/client/KillHyracksApplication.java     |   54 -
 .../yarn/client/LaunchHyracksApplication.java   |  101 --
 .../yarn/client/KillHyracksApplication.java     |   54 +
 .../yarn/client/LaunchHyracksApplication.java   |  101 ++
 .../common/protocols/amrm/AMRMConnection.java   |   59 -
 .../protocols/clientrm/YarnApplication.java     |   58 -
 .../clientrm/YarnClientRMConnection.java        |   65 -
 .../common/resources/LocalResourceHelper.java   |   52 -
 .../yarn/common/resources/ResourceHelper.java   |   26 -
 .../common/protocols/amrm/AMRMConnection.java   |   59 +
 .../protocols/clientrm/YarnApplication.java     |   58 +
 .../clientrm/YarnClientRMConnection.java        |   65 +
 .../common/resources/LocalResourceHelper.java   |   52 +
 .../yarn/common/resources/ResourceHelper.java   |   26 +
 3894 files changed, 195373 insertions(+), 195964 deletions(-)
----------------------------------------------------------------------

[48/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
deleted file mode 100644
index 24befcc..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-
-public interface IExpressionEvalSizeComputer {
-    // size in bytes, or -1 if unknown
-    public int getEvalSize(ILogicalExpression expr, IVariableEvalSizeEnvironment env) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
deleted file mode 100644
index f44cd29..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionRuntimeProvider.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
-
-public interface IExpressionRuntimeProvider {
-    public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
-            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
-
-    public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException;
-
-    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
-            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
-            JobGenContext context) throws AlgebricksException;
-
-    public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException;
-
-    public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException;
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
deleted file mode 100644
index 87c5ade..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public interface IExpressionTypeComputer {
-    public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
-            IVariableTypeEnvironment env) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
deleted file mode 100644
index 83e36b2..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
-
-public interface ILogicalExpressionJobGen {
-
-    public ICopyEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
-            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
-
-    public ICopyAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException;
-
-    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
-            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
-            JobGenContext context) throws AlgebricksException;
-
-    public ICopyRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException;
-
-    public ICopyUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
deleted file mode 100644
index 6a6989d..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IMergeAggregationExpressionFactory {
-    ILogicalExpression createMergeAggregation(LogicalVariable originalAggVariable, ILogicalExpression expr, IOptimizationContext env)
-            throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
deleted file mode 100644
index d3f6d19..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public interface INullableTypeComputer {
-    public Object makeNullableType(Object type) throws AlgebricksException;
-
-    public boolean canBeNull(Object type);
-
-    public Object getNonOptionalType(Object type);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
deleted file mode 100644
index 4d9fe38..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-public interface IPartialAggregationTypeComputer {
-    public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env,
-            IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
deleted file mode 100644
index 351f4a2..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IVariableEvalSizeEnvironment {
-    // size in bytes or -1 if unknown
-    public int getVariableEvalSize(LogicalVariable var);
-
-    public void setVariableEvalSize(LogicalVariable var, int size);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
deleted file mode 100644
index 8bcc575..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-
-public interface IVariableTypeEnvironment {
-    public Object getVarType(LogicalVariable var) throws AlgebricksException;
-
-    public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
-            List<List<LogicalVariable>> correlatedNullableVariableLists) throws AlgebricksException;
-
-    public void setVarType(LogicalVariable var, Object type);
-
-    public Object getType(ILogicalExpression expr) throws AlgebricksException;
-
-    public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
deleted file mode 100644
index 97a3f9a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IndexedNLJoinExpressionAnnotation.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-public class IndexedNLJoinExpressionAnnotation implements IExpressionAnnotation {
-
-    public static final String INDEXED_NL_JOIN_ANNOTATION_KEY = "indexnl";
-    public static final IndexedNLJoinExpressionAnnotation INSTANCE = new IndexedNLJoinExpressionAnnotation();
-
-    private Object object;
-
-    @Override
-    public Object getObject() {
-        return object;
-    }
-
-    @Override
-    public void setObject(Object object) {
-        this.object = object;
-    }
-
-    @Override
-    public IExpressionAnnotation copy() {
-        IndexedNLJoinExpressionAnnotation clone = new IndexedNLJoinExpressionAnnotation();
-        clone.setObject(object);
-        return clone;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
deleted file mode 100644
index 4241146..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunction;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyRunningAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopySerializableAggregateFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunction;
-import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyUnnestingFunctionFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IRunningAggregateEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluator;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.data.std.api.IPointable;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class LogicalExpressionJobGenToExpressionRuntimeProviderAdapter implements IExpressionRuntimeProvider {
-    private final ILogicalExpressionJobGen lejg;
-
-    public LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(ILogicalExpressionJobGen lejg) {
-        this.lejg = lejg;
-    }
-
-    @Override
-    public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
-            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
-        ICopyEvaluatorFactory cef = lejg.createEvaluatorFactory(expr, env, inputSchemas, context);
-        return new ScalarEvaluatorFactoryAdapter(cef);
-    }
-
-    @Override
-    public IAggregateEvaluatorFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException {
-        ICopyAggregateFunctionFactory caff = lejg.createAggregateFunctionFactory(expr, env, inputSchemas, context);
-        return new AggregateFunctionFactoryAdapter(caff);
-    }
-
-    @Override
-    public ICopySerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
-            AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
-            JobGenContext context) throws AlgebricksException {
-        return lejg.createSerializableAggregateFunctionFactory(expr, env, inputSchemas, context);
-    }
-
-    @Override
-    public IRunningAggregateEvaluatorFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException {
-        ICopyRunningAggregateFunctionFactory craff = lejg.createRunningAggregateFunctionFactory(expr, env,
-                inputSchemas, context);
-        return new RunningAggregateFunctionFactoryAdapter(craff);
-    }
-
-    @Override
-    public IUnnestingEvaluatorFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
-            IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
-            throws AlgebricksException {
-        ICopyUnnestingFunctionFactory cuff = lejg.createUnnestingFunctionFactory(expr, env, inputSchemas, context);
-        return new UnnestingFunctionFactoryAdapter(cuff);
-    }
-
-    public static final class ScalarEvaluatorFactoryAdapter implements IScalarEvaluatorFactory {
-        private static final long serialVersionUID = 1L;
-
-        private final ICopyEvaluatorFactory cef;
-
-        public ScalarEvaluatorFactoryAdapter(ICopyEvaluatorFactory cef) {
-            this.cef = cef;
-        }
-
-        @Override
-        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
-            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-            final ICopyEvaluator ce = cef.createEvaluator(abvs);
-            return new IScalarEvaluator() {
-                @Override
-                public void evaluate(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
-                    abvs.reset();
-                    ce.evaluate(tuple);
-                    result.set(abvs);
-                }
-            };
-        }
-    }
-
-    public static final class AggregateFunctionFactoryAdapter implements IAggregateEvaluatorFactory {
-        private static final long serialVersionUID = 1L;
-
-        private final ICopyAggregateFunctionFactory caff;
-
-        public AggregateFunctionFactoryAdapter(ICopyAggregateFunctionFactory caff) {
-            this.caff = caff;
-        }
-
-        @Override
-        public IAggregateEvaluator createAggregateEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
-            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-            final ICopyAggregateFunction caf = caff.createAggregateFunction(abvs);
-            return new IAggregateEvaluator() {
-                @Override
-                public void step(IFrameTupleReference tuple) throws AlgebricksException {
-                    caf.step(tuple);
-                }
-
-                @Override
-                public void init() throws AlgebricksException {
-                    abvs.reset();
-                    caf.init();
-                }
-
-                @Override
-                public void finishPartial(IPointable result) throws AlgebricksException {
-                    caf.finishPartial();
-                    result.set(abvs);
-                }
-
-                @Override
-                public void finish(IPointable result) throws AlgebricksException {
-                    caf.finish();
-                    result.set(abvs);
-                }
-
-            };
-        }
-    }
-
-    public static final class RunningAggregateFunctionFactoryAdapter implements IRunningAggregateEvaluatorFactory {
-        private static final long serialVersionUID = 1L;
-
-        private final ICopyRunningAggregateFunctionFactory craff;
-
-        public RunningAggregateFunctionFactoryAdapter(ICopyRunningAggregateFunctionFactory craff) {
-            this.craff = craff;
-        }
-
-        @Override
-        public IRunningAggregateEvaluator createRunningAggregateEvaluator() throws AlgebricksException {
-            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-            final ICopyRunningAggregateFunction craf = craff.createRunningAggregateFunction(abvs);
-            return new IRunningAggregateEvaluator() {
-                @Override
-                public void step(IFrameTupleReference tuple, IPointable result) throws AlgebricksException {
-                    abvs.reset();
-                    craf.step(tuple);
-                    result.set(abvs);
-                }
-
-                @Override
-                public void init() throws AlgebricksException {
-                    craf.init();
-                }
-            };
-        }
-    }
-
-    public static final class UnnestingFunctionFactoryAdapter implements IUnnestingEvaluatorFactory {
-        private static final long serialVersionUID = 1L;
-
-        private final ICopyUnnestingFunctionFactory cuff;
-
-        public UnnestingFunctionFactoryAdapter(ICopyUnnestingFunctionFactory cuff) {
-            this.cuff = cuff;
-        }
-
-        @Override
-        public IUnnestingEvaluator createUnnestingEvaluator(IHyracksTaskContext ctx) throws AlgebricksException {
-            final ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
-            final ICopyUnnestingFunction cuf = cuff.createUnnestingFunction(abvs);
-            return new IUnnestingEvaluator() {
-                @Override
-                public boolean step(IPointable result) throws AlgebricksException {
-                    abvs.reset();
-                    if (cuf.step()) {
-                        result.set(abvs);
-                        return true;
-                    }
-                    return false;
-                }
-
-                @Override
-                public void init(IFrameTupleReference tuple) throws AlgebricksException {
-                    cuf.init(tuple);
-                }
-            };
-        }
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
deleted file mode 100644
index 81f3dab..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-public class ScalarFunctionCallExpression extends AbstractFunctionCallExpression {
-
-    public ScalarFunctionCallExpression(IFunctionInfo finfo) {
-        super(FunctionKind.SCALAR, finfo);
-    }
-
-    public ScalarFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
-        super(FunctionKind.SCALAR, finfo, arguments);
-    }
-
-    public ScalarFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
-        super(FunctionKind.SCALAR, finfo, expressions);
-    }
-
-    @Override
-    public ScalarFunctionCallExpression cloneExpression() {
-        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
-        ScalarFunctionCallExpression funcExpr = new ScalarFunctionCallExpression(finfo, clonedArgs);
-        funcExpr.getAnnotations().putAll(cloneAnnotations());
-        funcExpr.setOpaqueParameters(this.getOpaqueParameters());
-        return funcExpr;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitScalarFunctionCallExpression(this, arg);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
deleted file mode 100644
index 63a5d1a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPropertiesComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-public class StatefulFunctionCallExpression extends AbstractFunctionCallExpression {
-
-    private final IPropertiesComputer propertiesComputer;
-
-    public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer) {
-        super(FunctionKind.STATEFUL, finfo);
-        this.propertiesComputer = propertiesComputer;
-    }
-
-    public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
-            List<Mutable<ILogicalExpression>> arguments) {
-        super(FunctionKind.STATEFUL, finfo, arguments);
-        this.propertiesComputer = propertiesComputer;
-    }
-
-    public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
-            Mutable<ILogicalExpression>... expressions) {
-        super(FunctionKind.STATEFUL, finfo, expressions);
-        this.propertiesComputer = propertiesComputer;
-    }
-
-    @Override
-    public StatefulFunctionCallExpression cloneExpression() {
-        cloneAnnotations();
-        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
-        return new StatefulFunctionCallExpression(finfo, propertiesComputer, clonedArgs);
-    }
-
-    // can be null
-    public IPartitioningProperty getRequiredPartitioningProperty() {
-        return propertiesComputer.computePartitioningProperty(this);
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitStatefulFunctionCallExpression(this, arg);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
deleted file mode 100644
index 562e8a1..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-public class UnnestingFunctionCallExpression extends AbstractFunctionCallExpression {
-
-    private boolean returnsUniqueValues;
-
-    public UnnestingFunctionCallExpression(IFunctionInfo finfo) {
-        super(FunctionKind.UNNEST, finfo);
-    }
-
-    public UnnestingFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
-        super(FunctionKind.UNNEST, finfo, arguments);
-    }
-
-    public UnnestingFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
-        super(FunctionKind.UNNEST, finfo, expressions);
-    }
-
-    @Override
-    public UnnestingFunctionCallExpression cloneExpression() {
-        cloneAnnotations();
-        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
-        UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(finfo, clonedArgs);
-        ufce.setReturnsUniqueValues(returnsUniqueValues);
-        ufce.setOpaqueParameters(this.getOpaqueParameters());
-        return ufce;
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitUnnestingFunctionCallExpression(this, arg);
-    }
-
-    public void setReturnsUniqueValues(boolean returnsUniqueValues) {
-        this.returnsUniqueValues = returnsUniqueValues;
-    }
-
-    public boolean returnsUniqueValues() {
-        return returnsUniqueValues;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
deleted file mode 100644
index 0ab1443..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
-
-public class VariableReferenceExpression extends AbstractLogicalExpression {
-    private int tupleRef;
-    private LogicalVariable variable;
-
-    public VariableReferenceExpression(int tupleRef, LogicalVariable variable) {
-        this.tupleRef = tupleRef;
-        this.variable = variable;
-    }
-
-    public VariableReferenceExpression(LogicalVariable variable) {
-        this(0, variable);
-    }
-
-    public int getTupleRef() {
-        return tupleRef;
-    }
-
-    public void setTupleRef(int tupleRef) {
-        this.tupleRef = tupleRef;
-    }
-
-    public LogicalVariable getVariableReference() {
-        return variable;
-    }
-
-    public void setVariable(LogicalVariable variable) {
-        this.variable = variable;
-    }
-
-    @Override
-    public LogicalExpressionTag getExpressionTag() {
-        return LogicalExpressionTag.VARIABLE;
-    }
-
-    @Override
-    public String toString() {
-        return "%" + tupleRef + "->" + variable.toString();
-    }
-
-    @Override
-    public void getUsedVariables(Collection<LogicalVariable> vars) {
-        // if (!vars.contains(variable)) {
-        vars.add(variable);
-        // }
-    }
-
-    @Override
-    public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
-        if (variable.equals(v1)) {
-            variable = v2;
-        }
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (!(obj instanceof VariableReferenceExpression)) {
-            return false;
-        } else {
-            return tupleRef == ((VariableReferenceExpression) obj).tupleRef
-                    && variable.equals(((VariableReferenceExpression) obj).getVariableReference());
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        return tupleRef + variable.getId();
-    }
-
-    @Override
-    public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
-        return visitor.visitVariableReferenceExpression(this, arg);
-    }
-
-    @Override
-    public AbstractLogicalExpression cloneExpression() {
-        return new VariableReferenceExpression(variable);
-    }
-
-    @Override
-    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
-        return false;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java
deleted file mode 100644
index b1d6ec6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AbstractFunctionInfo.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
-
-import java.io.Serializable;
-
-public abstract class AbstractFunctionInfo implements IFunctionInfo, Serializable {
-
-    private static final long serialVersionUID = 1L;
-
-    private final boolean isFunctional;
-
-    protected AbstractFunctionInfo(boolean isFunctional) {
-        this.isFunctional = isFunctional;
-    }
-
-    @Override
-    public boolean isFunctional() {
-        return isFunctional;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
deleted file mode 100644
index 5d75c5c..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class AlgebricksBuiltinFunctions {
-    public enum ComparisonKind {
-        EQ,
-        LE,
-        GE,
-        LT,
-        GT,
-        NEQ
-    }
-
-    public static final String ALGEBRICKS_NS = "algebricks";
-
-    // comparisons
-    public final static FunctionIdentifier EQ = new FunctionIdentifier(ALGEBRICKS_NS, "eq", 2);
-    public final static FunctionIdentifier LE = new FunctionIdentifier(ALGEBRICKS_NS, "le", 2);
-    public final static FunctionIdentifier GE = new FunctionIdentifier(ALGEBRICKS_NS, "ge", 2);
-    public final static FunctionIdentifier LT = new FunctionIdentifier(ALGEBRICKS_NS, "lt", 2);
-    public final static FunctionIdentifier GT = new FunctionIdentifier(ALGEBRICKS_NS, "gt", 2);
-    public final static FunctionIdentifier NEQ = new FunctionIdentifier(ALGEBRICKS_NS, "neq", 2);
-
-    // booleans
-    public final static FunctionIdentifier NOT = new FunctionIdentifier(ALGEBRICKS_NS, "not", 1);
-    public final static FunctionIdentifier AND = new FunctionIdentifier(ALGEBRICKS_NS, "and",
-            FunctionIdentifier.VARARGS);
-    public final static FunctionIdentifier OR = new FunctionIdentifier(ALGEBRICKS_NS, "or", FunctionIdentifier.VARARGS);
-
-    // numerics
-    public final static FunctionIdentifier NUMERIC_ADD = new FunctionIdentifier(ALGEBRICKS_NS, "numeric-add", 2);
-
-    // nulls
-    public final static FunctionIdentifier IS_NULL = new FunctionIdentifier(ALGEBRICKS_NS, "is-null", 1);
-
-    private static final Map<FunctionIdentifier, ComparisonKind> comparisonFunctions = new HashMap<FunctionIdentifier, ComparisonKind>();
-    static {
-        comparisonFunctions.put(AlgebricksBuiltinFunctions.EQ, ComparisonKind.EQ);
-        comparisonFunctions.put(AlgebricksBuiltinFunctions.LE, ComparisonKind.LE);
-        comparisonFunctions.put(AlgebricksBuiltinFunctions.GE, ComparisonKind.GE);
-        comparisonFunctions.put(AlgebricksBuiltinFunctions.LT, ComparisonKind.LT);
-        comparisonFunctions.put(AlgebricksBuiltinFunctions.GT, ComparisonKind.GT);
-        comparisonFunctions.put(AlgebricksBuiltinFunctions.NEQ, ComparisonKind.NEQ);
-    }
-
-    public static ComparisonKind getComparisonType(FunctionIdentifier fi) {
-        return comparisonFunctions.get(fi);
-    }
-
-    public static boolean isComparisonFunction(FunctionIdentifier fi) {
-        return comparisonFunctions.get(fi) != null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
deleted file mode 100644
index 7f812f3..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
-
-import java.io.Serializable;
-
-public final class FunctionIdentifier implements Serializable {
-    private static final long serialVersionUID = 1L;
-
-    private final String namespace;
-    private final String name;
-    private final int arity;
-
-    public final static int VARARGS = -1;
-
-    public FunctionIdentifier(String namespace, String name) {
-        this(namespace, name, VARARGS);
-    }
-
-    public FunctionIdentifier(String namespace, String name, int arity) {
-        this.namespace = namespace;
-        this.name = name;
-        this.arity = arity;
-    }
-
-    public String getName() {
-        return name;
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (super.equals(o)) {
-            return true;
-        }
-        if (o instanceof FunctionIdentifier) {
-            FunctionIdentifier ofi = (FunctionIdentifier) o;
-            return ofi.getNamespace().equals(getNamespace()) && ofi.name.equals(name);
-        }
-        return false;
-    }
-
-    @Override
-    public int hashCode() {
-        return name.hashCode() + namespace.hashCode();
-    }
-
-    public String toString() {
-        return getNamespace() + ":" + name;
-    }
-
-    public int getArity() {
-        return arity;
-    }
-
-    public String getNamespace() {
-        return namespace;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
deleted file mode 100644
index 02cd6de..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
-
-public interface IFunctionInfo {
-    FunctionIdentifier getFunctionIdentifier();
-
-    public boolean isFunctional();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java
deleted file mode 100644
index a7b7bc5..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-
-public interface IDataSink {
-    public Object getId();
-
-    public Object[] getSchemaTypes();
-
-    public IPartitioningProperty getPartitioningProperty();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java
deleted file mode 100644
index ed5cb4b..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-
-public interface IDataSource<T> {
-    public T getId();
-
-    public Object[] getSchemaTypes();
-
-    public IDataSourcePropertiesProvider getPropertiesProvider();
-
-    public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
deleted file mode 100644
index 20a2dcf..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
-
-public interface IDataSourceIndex<I, T> {
-    public I getId();
-
-    public IDataSource<T> getDataSource();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
deleted file mode 100644
index 3ae1e31..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-
-public interface IDataSourcePropertiesProvider {
-    public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
deleted file mode 100644
index c71b352..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public interface IMetadataProvider<S, I> {
-    public IDataSource<S> findDataSource(S id) throws AlgebricksException;
-
-    /**
-     * Obs: A scanner may choose to contribute a null
-     * AlgebricksPartitionConstraint and implement
-     * contributeSchedulingConstraints instead.
-     */
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
-            List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
-            List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
-            IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
-            throws AlgebricksException;
-
-    public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource);
-
-    public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
-            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
-            throws AlgebricksException;
-
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
-            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
-            JobSpecification spec) throws AlgebricksException;
-
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> dataSource,
-            IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
-            List<LogicalVariable> additionalNonKeyFields, JobGenContext context, JobSpecification jobSpec)
-            throws AlgebricksException;
-
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
-            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
-            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification jobSpec, boolean bulkload) throws AlgebricksException;
-
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
-            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
-            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
-
-    /**
-     * Creates the insert runtime of IndexInsertDeletePOperator, which models
-     * insert/delete operations into a secondary index.
-     * 
-     * @param dataSource
-     *            Target secondary index.
-     * @param propagatedSchema
-     *            Output schema of the insert/delete operator to be created.
-     * @param inputSchemas
-     *            Output schemas of the insert/delete operator to be created.
-     * @param typeEnv
-     *            Type environment of the original IndexInsertDeleteOperator operator.
-     * @param primaryKeys
-     *            Variables for the dataset's primary keys that the dataSource secondary index belongs to.
-     * @param secondaryKeys
-     *            Variables for the secondary-index keys.
-     * @param additionalNonKeyFields
-     *            Additional variables that can be passed to the secondary index as payload.
-     *            This can be useful when creating a second filter on a non-primary and non-secondary
-     *            fields for additional pruning power.
-     * @param filterExpr
-     *            Filtering expression to be pushed inside the runtime op.
-     *            Such a filter may, e.g., exclude NULLs from being inserted/deleted.
-     * @param recordDesc
-     *            Output record descriptor of the runtime op to be created.
-     * @param context
-     *            Job generation context.
-     * @param spec
-     *            Target job specification.
-     * @return
-     *         A Hyracks IOperatorDescriptor and its partition constraint.
-     * @throws AlgebricksException
-     */
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
-            IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
-            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException;
-
-    /**
-     * Creates the delete runtime of IndexInsertDeletePOperator, which models
-     * insert/delete operations into a secondary index.
-     * 
-     * @param dataSource
-     *            Target secondary index.
-     * @param propagatedSchema
-     *            Output schema of the insert/delete operator to be created.
-     * @param inputSchemas
-     *            Output schemas of the insert/delete operator to be created.
-     * @param typeEnv
-     *            Type environment of the original IndexInsertDeleteOperator operator.
-     * @param primaryKeys
-     *            Variables for the dataset's primary keys that the dataSource secondary index belongs to.
-     * @param secondaryKeys
-     *            Variables for the secondary-index keys.
-     * @param additionalNonKeyFields
-     *            Additional variables that can be passed to the secondary index as payload.
-     *            This can be useful when creating a second filter on a non-primary and non-secondary
-     *            fields for additional pruning power.
-     * @param filterExpr
-     *            Filtering expression to be pushed inside the runtime op.
-     *            Such a filter may, e.g., exclude NULLs from being inserted/deleted.
-     * @param recordDesc
-     *            Output record descriptor of the runtime op to be created.
-     * @param context
-     *            Job generation context.
-     * @param spec
-     *            Target job specification.
-     * @return
-     *         A Hyracks IOperatorDescriptor and its partition constraint.
-     * @throws AlgebricksException
-     */
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
-            IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
-            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            List<LogicalVariable> additionalNonKeyFields, ILogicalExpression filterExpr, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification spec) throws AlgebricksException;
-
-    /**
-     * Creates the TokenizeOperator for IndexInsertDeletePOperator, which tokenizes
-     * secondary key into [token, number of token] pair in a length-partitioned index.
-     * In case of non length-partitioned index, it tokenizes secondary key into [token].
-     * 
-     * @param dataSource
-     *            Target secondary index.
-     * @param propagatedSchema
-     *            Output schema of the insert/delete operator to be created.
-     * @param inputSchemas
-     *            Output schemas of the insert/delete operator to be created.
-     * @param typeEnv
-     *            Type environment of the original IndexInsertDeleteOperator operator.
-     * @param primaryKeys
-     *            Variables for the dataset's primary keys that the dataSource secondary index belongs to.
-     * @param secondaryKeys
-     *            Variables for the secondary-index keys.
-     * @param filterExpr
-     *            Filtering expression to be pushed inside the runtime op.
-     *            Such a filter may, e.g., exclude NULLs from being inserted/deleted.
-     * @param recordDesc
-     *            Output record descriptor of the runtime op to be created.
-     * @param context
-     *            Job generation context.
-     * @param spec
-     *            Target job specification.
-     * @return
-     *         A Hyracks IOperatorDescriptor and its partition constraint.
-     * @throws AlgebricksException
-     */
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(
-            IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
-            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec,
-            boolean bulkload) throws AlgebricksException;
-
-    public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
-
-    public IFunctionInfo lookupFunction(FunctionIdentifier fid);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
deleted file mode 100644
index 687025a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-
-/**
- * @author Nicola
- */
-public abstract class AbstractAssignOperator extends AbstractLogicalOperator {
-    protected final List<LogicalVariable> variables;
-    protected final List<Mutable<ILogicalExpression>> expressions;
-
-    public AbstractAssignOperator() {
-        this.variables = new ArrayList<LogicalVariable>();
-        this.expressions = new ArrayList<Mutable<ILogicalExpression>>();
-    }
-
-    public AbstractAssignOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
-        this.variables = variables;
-        this.expressions = expressions;
-    }
-
-    public List<LogicalVariable> getVariables() {
-        return variables;
-    }
-
-    public List<Mutable<ILogicalExpression>> getExpressions() {
-        return expressions;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-        schema.addAll(variables);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        boolean modif = false;
-        for (int i = 0; i < expressions.size(); i++) {
-            if (visitor.transform(expressions.get(i))) {
-                modif = true;
-            }
-        }
-        return modif;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
deleted file mode 100644
index 6c00412..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.ArrayList;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
-
-public abstract class AbstractBinaryJoinOperator extends AbstractLogicalOperator {
-    protected final Mutable<ILogicalExpression> condition;
-    protected JoinKind joinKind;
-
-    public enum JoinKind {
-        INNER,
-        LEFT_OUTER
-    }
-
-    public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition) {
-        this.joinKind = joinKind;
-        this.condition = condition;
-    }
-
-    public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition,
-            Mutable<ILogicalOperator> input1, Mutable<ILogicalOperator> input2) {
-        this(joinKind, condition);
-        inputs.add(input1);
-        inputs.add(input2);
-    }
-
-    public Mutable<ILogicalExpression> getCondition() {
-        return condition;
-    }
-
-    public JoinKind getJoinKind() {
-        return joinKind;
-    }
-
-    @Override
-    public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getValue().getSchema());
-        schema.addAll(inputs.get(1).getValue().getSchema());
-    }
-
-    @Override
-    public VariablePropagationPolicy getVariablePropagationPolicy() {
-        return VariablePropagationPolicy.concat(VariablePropagationPolicy.ALL, VariablePropagationPolicy.ALL);
-    }
-
-    @Override
-    public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
-        return visitor.transform(condition);
-    }
-
-    @Override
-    public boolean isMap() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java
deleted file mode 100644
index 85660e2..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractDataSourceOperator.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
-
-import java.util.List;
-
-public abstract class AbstractDataSourceOperator extends AbstractScanOperator {
-    protected IDataSource<?> dataSource;
-
-    public AbstractDataSourceOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
-        super(variables);
-        this.dataSource = dataSource;
-    }
-
-    public IDataSource<?> getDataSource() {
-        return dataSource;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
deleted file mode 100644
index 2b3c60a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractExtensibleLogicalOperator.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-
-/**
- * @author rico
- */
-public abstract class AbstractExtensibleLogicalOperator implements IOperatorExtension {
-
-    private AbstractLogicalOperator.ExecutionMode mode = AbstractLogicalOperator.ExecutionMode.UNPARTITIONED;
-    protected List<LogicalVariable> schema;
-    protected IPhysicalOperator physicalOperator;
-
-    @Override
-    public ExecutionMode getExecutionMode() {
-        return mode;
-    }
-
-    @Override
-    public void setExecutionMode(ExecutionMode mode) {
-        this.mode = mode;
-    }
-
-    @Override
-    public void setSchema(List<LogicalVariable> schema) {
-        this.schema = schema;
-    }
-
-    @Override
-    public IPhysicalOperator getPhysicalOperator() {
-        return physicalOperator;
-    }
-
-    @Override
-    public void setPhysicalOperator(IPhysicalOperator physicalOperator) {
-        this.physicalOperator = physicalOperator;
-    }
-}
\ No newline at end of file

[43/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
deleted file mode 100644
index e62dbf8..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/OperatorDeepCopyVisitor.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class OperatorDeepCopyVisitor implements ILogicalOperatorVisitor<ILogicalOperator, Void> {
-
-    @Override
-    public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        newList.addAll(op.getVariables());
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new AggregateOperator(newList, newExpressions);
-    }
-
-    @Override
-    public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
-            throws AlgebricksException {
-        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        newList.addAll(op.getVariables());
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new RunningAggregateOperator(newList, newExpressions);
-    }
-
-    @Override
-    public ILogicalOperator visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg)
-            throws AlgebricksException {
-        return new EmptyTupleSourceOperator();
-    }
-
-    @Override
-    public ILogicalOperator visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
-        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decoList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
-        ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getGroupByList())
-            groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
-                    deepCopyExpressionRef(pair.second)));
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList())
-            decoList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
-                    deepCopyExpressionRef(pair.second)));
-        for (ILogicalPlan plan : op.getNestedPlans()) {
-            newSubplans.add(OperatorManipulationUtil.deepCopy(plan));
-        }
-        return new GroupByOperator(groupByList, decoList, newSubplans);
-    }
-
-    @Override
-    public ILogicalOperator visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
-        return new LimitOperator(deepCopyExpressionRef(op.getMaxObjects()).getValue(), deepCopyExpressionRef(
-                op.getOffset()).getValue(), op.isTopmostLimitOp());
-    }
-
-    @Override
-    public ILogicalOperator visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
-        return new InnerJoinOperator(deepCopyExpressionRef(op.getCondition()), op.getInputs().get(0), op.getInputs()
-                .get(1));
-    }
-
-    @Override
-    public ILogicalOperator visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
-        return new LeftOuterJoinOperator(deepCopyExpressionRef(op.getCondition()), op.getInputs().get(0), op
-                .getInputs().get(1));
-    }
-
-    @Override
-    public ILogicalOperator visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg)
-            throws AlgebricksException {
-        return new NestedTupleSourceOperator(null);
-    }
-
-    @Override
-    public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
-        return new OrderOperator(this.deepCopyOrderAndExpression(op.getOrderExpressions()));
-    }
-
-    @Override
-    public ILogicalOperator visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        newList.addAll(op.getVariables());
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new AssignOperator(newList, newExpressions);
-    }
-
-    @Override
-    public ILogicalOperator visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
-        return new SelectOperator(deepCopyExpressionRef(op.getCondition()), op.getRetainNull(),
-                op.getNullPlaceholderVariable());
-    }
-
-    @Override
-    public ILogicalOperator visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-        newList.addAll(op.getVariables());
-        return new ProjectOperator(newList);
-    }
-
-    @Override
-    public ILogicalOperator visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg)
-            throws AlgebricksException {
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new PartitioningSplitOperator(newExpressions, op.getDefaultBranchIndex());
-    }
-
-    @Override
-    public ILogicalOperator visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        return new ReplicateOperator(op.getOutputArity());
-    }
-
-    @Override
-    public ILogicalOperator visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
-        ArrayList<LogicalVariable> newOutputList = new ArrayList<LogicalVariable>();
-        newInputList.addAll(op.getInputVariables());
-        newOutputList.addAll(op.getOutputVariables());
-        return new ScriptOperator(op.getScriptDescription(), newInputList, newOutputList);
-    }
-
-    @Override
-    public ILogicalOperator visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
-        for (ILogicalPlan plan : op.getNestedPlans()) {
-            newSubplans.add(OperatorManipulationUtil.deepCopy(plan));
-        }
-        return new SubplanOperator(newSubplans);
-    }
-
-    @Override
-    public ILogicalOperator visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> newVarMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>();
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple : varMap)
-            newVarMap.add(new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(triple.first, triple.second,
-                    triple.third));
-        return new UnionAllOperator(newVarMap);
-    }
-
-    @Override
-    public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
-        return new UnnestOperator(op.getVariable(), deepCopyExpressionRef(op.getExpressionRef()),
-                op.getPositionalVariable(), op.getPositionalVariableType(), op.getPositionWriter());
-    }
-
-    @Override
-    public ILogicalOperator visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
-        newInputList.addAll(op.getVariables());
-        return new UnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()), new ArrayList<Object>(
-                op.getVariableTypes()), op.propagatesInput());
-    }
-
-    @Override
-    public ILogicalOperator visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
-        newInputList.addAll(op.getVariables());
-        return new DataSourceScanOperator(newInputList, op.getDataSource());
-    }
-
-    @Override
-    public ILogicalOperator visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new DistinctOperator(newExpressions);
-    }
-
-    @Override
-    public ILogicalOperator visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        return new ExchangeOperator();
-    }
-
-    @Override
-    public ILogicalOperator visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new WriteOperator(newExpressions, op.getDataSink());
-    }
-
-    @Override
-    public ILogicalOperator visitDistributeResultOperator(DistributeResultOperator op, Void arg)
-            throws AlgebricksException {
-        ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newExpressions, op.getExpressions());
-        return new DistributeResultOperator(newExpressions, op.getDataSink());
-    }
-
-    @Override
-    public ILogicalOperator visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
-        ArrayList<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newKeyExpressions, op.getKeyExpressions());
-        List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
-        WriteResultOperator writeResultOp = new WriteResultOperator(op.getDataSource(),
-                deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions);
-        writeResultOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
-        return writeResultOp;
-    }
-
-    @Override
-    public ILogicalOperator visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
-        List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions());
-        List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newKeyExpressions, op.getAdditionalFilteringExpressions());
-        InsertDeleteOperator insertDeleteOp = new InsertDeleteOperator(op.getDataSource(),
-                deepCopyExpressionRef(op.getPayloadExpression()), newKeyExpressions, op.getOperation(), op.isBulkload());
-        insertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
-        return insertDeleteOp;
-    }
-
-    @Override
-    public ILogicalOperator visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg)
-            throws AlgebricksException {
-        List<Mutable<ILogicalExpression>> newPrimaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
-        List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
-        Mutable<ILogicalExpression> newFilterExpression = new MutableObject<ILogicalExpression>(
-                ((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression());
-        List<Mutable<ILogicalExpression>> newLSMComponentFilterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newLSMComponentFilterExpressions, op.getAdditionalFilteringExpressions());
-        IndexInsertDeleteOperator indexInsertDeleteOp = new IndexInsertDeleteOperator(op.getDataSourceIndex(),
-                newPrimaryKeyExpressions, newSecondaryKeyExpressions, newFilterExpression, op.getOperation(),
-                op.isBulkload());
-        indexInsertDeleteOp.setAdditionalFilteringExpressions(newLSMComponentFilterExpressions);
-        return indexInsertDeleteOp;
-    }
-
-    @Override
-    public ILogicalOperator visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
-        List<Mutable<ILogicalExpression>> newPrimaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
-        List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-        deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
-        List<LogicalVariable> newTokenizeVars = new ArrayList<LogicalVariable>();
-        deepCopyVars(newTokenizeVars, op.getTokenizeVars());
-        Mutable<ILogicalExpression> newFilterExpression = new MutableObject<ILogicalExpression>(
-                ((AbstractLogicalExpression) op.getFilterExpression()).cloneExpression());
-        List<Object> newTokenizeVarTypes = new ArrayList<Object>();
-        deepCopyObjects(newTokenizeVarTypes, op.getTokenizeVarTypes());
-
-        TokenizeOperator tokenizeOp = new TokenizeOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
-                newSecondaryKeyExpressions, newTokenizeVars, newFilterExpression, op.getOperation(), op.isBulkload(),
-                op.isPartitioned(), newTokenizeVarTypes);
-        return tokenizeOp;
-    }
-
-    @Override
-    public ILogicalOperator visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
-        return new SinkOperator();
-    }
-
-    private void deepCopyExpressionRefs(List<Mutable<ILogicalExpression>> newExprs,
-            List<Mutable<ILogicalExpression>> oldExprs) {
-        for (Mutable<ILogicalExpression> oldExpr : oldExprs)
-            newExprs.add(new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue())
-                    .cloneExpression()));
-    }
-
-    private Mutable<ILogicalExpression> deepCopyExpressionRef(Mutable<ILogicalExpression> oldExpr) {
-        return new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression());
-    }
-
-    private List<LogicalVariable> deepCopyVars(List<LogicalVariable> newVars, List<LogicalVariable> oldVars) {
-        for (LogicalVariable oldVar : oldVars)
-            newVars.add(oldVar);
-        return newVars;
-    }
-
-    private List<Object> deepCopyObjects(List<Object> newObjs, List<Object> oldObjs) {
-        for (Object oldObj : oldObjs)
-            newObjs.add(oldObj);
-        return newObjs;
-    }
-
-    private List<Pair<IOrder, Mutable<ILogicalExpression>>> deepCopyOrderAndExpression(
-            List<Pair<IOrder, Mutable<ILogicalExpression>>> ordersAndExprs) {
-        List<Pair<IOrder, Mutable<ILogicalExpression>>> newOrdersAndExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
-        for (Pair<IOrder, Mutable<ILogicalExpression>> pair : ordersAndExprs)
-            newOrdersAndExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(pair.first,
-                    deepCopyExpressionRef(pair.second)));
-        return newOrdersAndExprs;
-    }
-
-    @Override
-    public ILogicalOperator visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        return new ExtensionOperator(op.getNewInstanceOfDelegateOperator());
-    }
-
-    @Override
-    public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
-            throws AlgebricksException {
-        ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
-        newInputList.addAll(op.getVariables());
-        return new ExternalDataLookupOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()),
-                new ArrayList<Object>(op.getVariableTypes()), op.isPropagateInput(), op.getDataSource());
-    }
-
-    @Override
-    public ILogicalOperator visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
-        return new MaterializeOperator();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
deleted file mode 100644
index 1fa97a6..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class ProducedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
-    private Collection<LogicalVariable> producedVariables;
-
-    public ProducedVariableVisitor(Collection<LogicalVariable> producedVariables) throws AlgebricksException {
-        this.producedVariables = producedVariables;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
-            if (p.first != null) {
-                producedVariables.add(p.first);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
-            if (p.first != null) {
-                producedVariables.add(p.first);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
-        List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op, usedVariables);
-        for (LogicalVariable v : op.getOutputVariables()) {
-            if (!usedVariables.contains(v)) {
-                producedVariables.add(v);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : op.getVariableMappings()) {
-            producedVariables.add(t.third);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getVariables());
-        LogicalVariable positionalVariable = op.getPositionalVariable();
-        if (positionalVariable != null) {
-            if (!producedVariables.contains(positionalVariable))
-                producedVariables.add(positionalVariable);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
-        producedVariables.addAll(op.getTokenizeVars());
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        op.getDelegate().getProducedVariables(producedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
-        producedVariables.add(op.getVariables().get(0));
-        return null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
deleted file mode 100644
index 0b37b92..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SchemaVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
-    private Collection<LogicalVariable> schemaVariables;
-
-    public SchemaVariableVisitor(Collection<LogicalVariable> schemaVariables) {
-        this.schemaVariables = schemaVariables;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
-        schemaVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
-        List<LogicalVariable> allLiveVars = new ArrayList<LogicalVariable>();
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), allLiveVars);
-        }
-        VariableUtilities.getProducedVariables(op, allLiveVars);
-        /** put distinct vars first */
-        schemaVariables.addAll(op.getDistinctByVarList());
-        /** then other live vars */
-        for (LogicalVariable var : allLiveVars) {
-            if (!schemaVariables.contains(var)) {
-                schemaVariables.add(var);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
-            if (p.first != null) {
-                schemaVariables.add(p.first);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
-            if (p.first != null) {
-                schemaVariables.add(p.first);
-            } else {
-                ILogicalExpression e = p.second.getValue();
-                if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                    schemaVariables.add(((VariableReferenceExpression) e).getVariableReference());
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
-        VariableUtilities.getLiveVariables(op.getSourceOperator(), schemaVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
-        schemaVariables.addAll(op.getVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
-        // VariableUtilities.getProducedVariables(op, schemaVariables);
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
-        schemaVariables.addAll(op.getOutputVariables());
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
-        }
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
-        if (op.propagatesInput()) {
-            standardLayout(op);
-        } else {
-            VariableUtilities.getProducedVariables(op, schemaVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    private void standardLayout(ILogicalOperator op) throws AlgebricksException {
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
-        }
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        standardLayout(op);
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
-        ArrayList<LogicalVariable> liveVariables = new ArrayList<LogicalVariable>();
-        ArrayList<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
-        //get used variables
-        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
-        //live variables - used variables
-        for (Mutable<ILogicalOperator> c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getValue(), liveVariables);
-        }
-        for (LogicalVariable v : liveVariables) {
-            if (!usedVariables.contains(v)) {
-                schemaVariables.add(v);
-            }
-        }
-        VariableUtilities.getProducedVariables(op, schemaVariables);
-        //+ produced variables
-        return null;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
deleted file mode 100644
index 695078a..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ /dev/null
@@ -1,469 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class SubstituteVariableVisitor implements ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> {
-
-    private final boolean goThroughNts;
-    private final ITypingContext ctx;
-
-    public SubstituteVariableVisitor(boolean goThroughNts, ITypingContext ctx) {
-        this.goThroughNts = goThroughNts;
-        this.ctx = ctx;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            if (variables.get(i).equals(pair.first)) {
-                variables.set(i, pair.second);
-            } else {
-                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            if (variables.get(i).equals(pair.first)) {
-                variables.set(i, pair.second);
-            } else {
-                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
-            }
-        }
-        // Substitute variables stored in ordering property
-        if (op.getExplicitOrderingProperty() != null) {
-            List<OrderColumn> orderColumns = op.getExplicitOrderingProperty().getOrderColumns();
-            for (int i = 0; i < orderColumns.size(); i++) {
-                OrderColumn oc = orderColumns.get(i);
-                if (oc.getColumn().equals(pair.first)) {
-                    orderColumns.set(i, new OrderColumn(pair.second, oc.getOrder()));
-                }
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
-            eRef.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        subst(pair.first, pair.second, op.getGroupByList());
-        subst(pair.first, pair.second, op.getDecorList());
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
-                        pair.second, goThroughNts, ctx);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getCondition().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getCondition().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
-        ILogicalExpression offset = op.getOffset().getValue();
-        if (offset != null) {
-            offset.substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
-            oe.second.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> usedVariables = op.getVariables();
-        int n = usedVariables.size();
-        for (int i = 0; i < n; i++) {
-            LogicalVariable v = usedVariables.get(i);
-            if (v.equals(pair.first)) {
-                usedVariables.set(i, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        int n = variables.size();
-        for (int i = 0; i < n; i++) {
-            if (variables.get(i).equals(pair.first)) {
-                variables.set(i, pair.second);
-            } else {
-                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        substInArray(op.getInputVariables(), pair.first, pair.second);
-        substInArray(op.getOutputVariables(), pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        op.getCondition().getValue().substituteVar(pair.first, pair.second);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
-                        pair.second, goThroughNts, ctx);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
-            if (t.first.equals(pair.first)) {
-                t.first = pair.second;
-            }
-            if (t.second.equals(pair.first)) {
-                t.second = pair.second;
-            }
-            if (t.third.equals(pair.first)) {
-                t.third = pair.second;
-            }
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
-        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    private void subst(LogicalVariable v1, LogicalVariable v2,
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> varExprPairList) {
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : varExprPairList) {
-            if (ve.first != null && ve.first.equals(v1)) {
-                ve.first = v2;
-                return;
-            }
-            ve.second.getValue().substituteVar(v1, v2);
-        }
-    }
-
-    private void substInArray(ArrayList<LogicalVariable> varArray, LogicalVariable v1, LogicalVariable v2) {
-        for (int i = 0; i < varArray.size(); i++) {
-            LogicalVariable v = varArray.get(i);
-            if (v == v1) {
-                varArray.set(i, v2);
-            }
-        }
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        op.substituteVar(arg.first, arg.second);
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
-            e.getValue().substituteVar(pair.first, pair.second);
-        }
-        substVarTypes(op, pair);
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Pair<LogicalVariable, LogicalVariable> pair)
-            throws AlgebricksException {
-        return null;
-    }
-
-    private void substVarTypes(ILogicalOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        if (ctx == null) {
-            return;
-        }
-        IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(op);
-        env.substituteProducedVariable(arg.first, arg.second);
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Pair<LogicalVariable, LogicalVariable> arg)
-            throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op,
-            Pair<LogicalVariable, LogicalVariable> pair) throws AlgebricksException {
-        List<LogicalVariable> variables = op.getVariables();
-        for (int i = 0; i < variables.size(); i++) {
-            if (variables.get(i) == pair.first) {
-                variables.set(i, pair.second);
-                return null;
-            }
-        }
-        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
-        substVarTypes(op, pair);
-        return null;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
deleted file mode 100644
index 8169ad0..0000000
--- a/algebricks/algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ /dev/null
@@ -1,407 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
-
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistributeResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionMergePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
-
-public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
-
-    private Collection<LogicalVariable> usedVariables;
-
-    public UsedVariableVisitor(Collection<LogicalVariable> usedVariables) {
-        this.usedVariables = usedVariables;
-    }
-
-    @Override
-    public Void visitAggregateOperator(AggregateOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-            exprRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitAssignOperator(AssignOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-            exprRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) {
-        if (op.getAdditionalFilteringExpressions() != null) {
-            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
-                e.getValue().getUsedVariables(usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitDistinctOperator(DistinctOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
-            eRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
-        // Used variables depend on the physical operator.
-        if (op.getPhysicalOperator() != null) {
-            IPhysicalOperator physOp = op.getPhysicalOperator();
-            switch (physOp.getOperatorTag()) {
-                case BROADCAST_EXCHANGE:
-                case ONE_TO_ONE_EXCHANGE:
-                case RANDOM_MERGE_EXCHANGE: {
-                    // No variables used.
-                    break;
-                }
-                case HASH_PARTITION_EXCHANGE: {
-                    HashPartitionExchangePOperator concreteOp = (HashPartitionExchangePOperator) physOp;
-                    usedVariables.addAll(concreteOp.getHashFields());
-                    break;
-                }
-                case HASH_PARTITION_MERGE_EXCHANGE: {
-                    HashPartitionMergeExchangePOperator concreteOp = (HashPartitionMergeExchangePOperator) physOp;
-                    usedVariables.addAll(concreteOp.getPartitionFields());
-                    for (OrderColumn orderCol : concreteOp.getOrderColumns()) {
-                        usedVariables.add(orderCol.getColumn());
-                    }
-                    break;
-                }
-                case SORT_MERGE_EXCHANGE: {
-                    SortMergeExchangePOperator concreteOp = (SortMergeExchangePOperator) physOp;
-                    for (OrderColumn orderCol : concreteOp.getSortColumns()) {
-                        usedVariables.add(orderCol.getColumn());
-                    }
-                    break;
-                }
-                case RANGE_PARTITION_EXCHANGE: {
-                    RangePartitionPOperator concreteOp = (RangePartitionPOperator) physOp;
-                    for (OrderColumn partCol : concreteOp.getPartitioningFields()) {
-                        usedVariables.add(partCol.getColumn());
-                    }
-                    break;
-                }
-                case RANGE_PARTITION_MERGE_EXCHANGE: {
-                    RangePartitionMergePOperator concreteOp = (RangePartitionMergePOperator) physOp;
-                    for (OrderColumn partCol : concreteOp.getPartitioningFields()) {
-                        usedVariables.add(partCol.getColumn());
-                    }
-                    break;
-                }
-                default: {
-                    throw new AlgebricksException("Unhandled physical operator tag '" + physOp.getOperatorTag() + "'.");
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
-            }
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) {
-            g.second.getValue().getUsedVariables(usedVariables);
-        }
-        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) {
-            g.second.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) {
-        op.getCondition().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) {
-        op.getCondition().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitLimitOperator(LimitOperator op, Void arg) {
-        op.getMaxObjects().getValue().getUsedVariables(usedVariables);
-        ILogicalExpression offsetExpr = op.getOffset().getValue();
-        if (offsetExpr != null) {
-            offsetExpr.getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) {
-        // does not use any variable
-        return null;
-    }
-
-    @Override
-    public Void visitOrderOperator(OrderOperator op, Void arg) {
-        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
-            oe.second.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitProjectOperator(ProjectOperator op, Void arg) {
-        List<LogicalVariable> parameterVariables = op.getVariables();
-        for (LogicalVariable v : parameterVariables) {
-            if (!usedVariables.contains(v)) {
-                usedVariables.add(v);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
-            exprRef.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitScriptOperator(ScriptOperator op, Void arg) {
-        List<LogicalVariable> parameterVariables = op.getInputVariables();
-        for (LogicalVariable v : parameterVariables) {
-            if (!usedVariables.contains(v)) {
-                usedVariables.add(v);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitSelectOperator(SelectOperator op, Void arg) {
-        op.getCondition().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (ILogicalPlan p : op.getNestedPlans()) {
-            for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnionOperator(UnionAllOperator op, Void arg) {
-        for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> m : op.getVariableMappings()) {
-            if (!usedVariables.contains(m.first)) {
-                usedVariables.add(m.first);
-            }
-            if (!usedVariables.contains(m.second)) {
-                usedVariables.add(m.second);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
-        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
-        if (op.getAdditionalFilteringExpressions() != null) {
-            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
-                e.getValue().getUsedVariables(usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitUnnestOperator(UnnestOperator op, Void arg) {
-        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitWriteOperator(WriteOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
-            expr.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitDistributeResultOperator(DistributeResultOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
-            expr.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitWriteResultOperator(WriteResultOperator op, Void arg) {
-        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
-        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        if (op.getAdditionalFilteringExpressions() != null) {
-            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
-                e.getValue().getUsedVariables(usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) {
-        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        if (op.getAdditionalFilteringExpressions() != null) {
-            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
-                e.getValue().getUsedVariables(usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        if (op.getAdditionalFilteringExpressions() != null) {
-            for (Mutable<ILogicalExpression> e : op.getAdditionalFilteringExpressions()) {
-                e.getValue().getUsedVariables(usedVariables);
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitTokenizeOperator(TokenizeOperator op, Void arg) {
-        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
-            e.getValue().getUsedVariables(usedVariables);
-        }
-        return null;
-    }
-
-    @Override
-    public Void visitSinkOperator(SinkOperator op, Void arg) {
-        return null;
-    }
-
-    @Override
-    public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitMaterializeOperator(MaterializeOperator op, Void arg) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Void visitExtensionOperator(ExtensionOperator op, Void arg) throws AlgebricksException {
-        op.getDelegate().getUsedVariables(usedVariables);
-        return null;
-    }
-
-    @Override
-    public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg) throws AlgebricksException {
-        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
-        return null;
-    }
-
-}

[13/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
deleted file mode 100644
index 8929623..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-import java.util.ListIterator;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-
-public class IntroJoinInsideSubplanRule extends AbstractDecorrelationRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
-        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op0;
-
-        Mutable<ILogicalOperator> leftRef = subplan.getInputs().get(0);
-        if (((AbstractLogicalOperator) leftRef.getValue()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-            return false;
-        }
-
-        ListIterator<ILogicalPlan> plansIter = subplan.getNestedPlans().listIterator();
-        ILogicalPlan p = null;
-        while (plansIter.hasNext()) {
-            p = plansIter.next();
-        }
-        if (p == null) {
-            return false;
-        }
-        if (p.getRoots().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> opRef1 = p.getRoots().get(0);
-
-        while (true) {
-            AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getValue();
-            if (op1.getInputs().size() != 1) {
-                return false;
-            }
-            if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
-                Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-                if (op2.getOperatorTag() != LogicalOperatorTag.SELECT && descOrSelfIsScanOrJoin(op2)) {
-                    Set<LogicalVariable> free2 = new HashSet<LogicalVariable>();
-                    OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, free2);
-                    if (free2.isEmpty()) {
-                        Set<LogicalVariable> free1 = new HashSet<LogicalVariable>();
-                        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op1, free1);
-                        if (!free1.isEmpty()) {
-                            OperatorManipulationUtil.ntsToEts(op2Ref, context);
-                            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(
-                                    new MutableObject<ILogicalOperator>(subplan));
-                            Mutable<ILogicalOperator> ntsRef = new MutableObject<ILogicalOperator>(nts);
-                            Mutable<ILogicalOperator> innerRef = new MutableObject<ILogicalOperator>(op2);
-                            InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(
-                                    ConstantExpression.TRUE), ntsRef, innerRef);
-                            op2Ref.setValue(join);
-                            context.computeAndSetTypeEnvironmentForOperator(nts);
-                            context.computeAndSetTypeEnvironmentForOperator(join);
-                            return true;
-                        }
-                    }
-                }
-            }
-            opRef1 = op1.getInputs().get(0);
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
deleted file mode 100644
index b0dbb1e..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceAggregateCombinerRule.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-
-public class IntroduceAggregateCombinerRule extends AbstractIntroduceCombinerRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (context.checkIfInDontApplySet(this, op)) {
-            return false;
-        }
-        context.addToDontApplySet(this, op);
-        if (op.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-        AggregateOperator aggOp = (AggregateOperator) op;
-        if (!aggOp.isGlobal() || aggOp.getExecutionMode() == ExecutionMode.LOCAL) {
-            return false;
-        }
-        Set<SimilarAggregatesInfo> toReplaceSet = new HashSet<SimilarAggregatesInfo>();
-        Pair<Boolean, Mutable<ILogicalOperator>> result = tryToPushAgg(aggOp, null, toReplaceSet, context);
-        if (!result.first || result.second == null) {
-            return false;
-        }
-        replaceOriginalAggFuncs(toReplaceSet);
-        context.computeAndSetTypeEnvironmentForOperator(aggOp);
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
deleted file mode 100644
index aa418ea..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByCombinerRule.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-
-public class IntroduceGroupByCombinerRule extends AbstractIntroduceGroupByCombinerRule {
-
-    @Override
-    protected void processNullTest(IOptimizationContext context, GroupByOperator nestedGby,
-            List<LogicalVariable> aggregateVarsProducedByCombiner) {
-        /** The default introduce group-by combiner rule ignores null test, however a language implementor can decide their own semantics. */
-    }
-
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
deleted file mode 100644
index a6d9ab0..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.ListSet;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
-
-/**
- * The rule searches for SUBPLAN operator with a optional PROJECT operator and
- * an AGGREGATE followed by a join operator.
- *
- * <pre>
- * Before
- * 
- *   plan__parent
- *   SUBPLAN {
- *     PROJECT?
- *     AGGREGATE
- *     plan__nested_A
- *     INNER_JOIN | LEFT_OUTER_JOIN ($condition, $left, $right)
- *       plan__nested_B
- *   }
- *   plan__child
- * 
- *   where $condition does not equal a constant true.
- * 
- * After (This is a general application of the rule, specifics may vary based on the query plan.)
- * 
- *   plan__parent
- *   GROUP_BY {
- *     PROJECT?
- *     AGGREGATE
- *     plan__nested_A
- *     SELECT( algebricks:not( is_null( $right ) ) )
- *     NESTED_TUPLE_SOURCE
- *   }
- *   SUBPLAN {
- *     INNER_JOIN | LEFT_OUTER_JOIN ($condition, $left, $right)
- *       plan__nested_B
- *   }
- *   plan__child
- * </pre>
- *
- * @author prestonc
- */
-
-public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
-        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op0;
-
-        Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
-        ILogicalPlan p = null;
-        while (plansIter.hasNext()) {
-            p = plansIter.next();
-        }
-        if (p == null) {
-            return false;
-        }
-        if (p.getRoots().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
-
-        Mutable<ILogicalOperator> botRef = subplanRoot;
-        AbstractLogicalOperator op2;
-        // Project is optional
-        if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
-            op2 = op1;
-        } else {
-            ProjectOperator project = (ProjectOperator) op1;
-            botRef = project.getInputs().get(0);
-            op2 = (AbstractLogicalOperator) botRef.getValue();
-        }
-        if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-            return false;
-        }
-        AggregateOperator aggregate = (AggregateOperator) op2;
-
-        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
-        VariableUtilities.getUsedVariables(aggregate, free);
-
-        Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
-
-        while (op3.getInputs().size() == 1) {
-            Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
-            VariableUtilities.getProducedVariables(op3, prod);
-            free.removeAll(prod);
-            VariableUtilities.getUsedVariables(op3, free);
-            botRef = op3Ref;
-            op3Ref = op3.getInputs().get(0);
-            op3 = (AbstractLogicalOperator) op3Ref.getValue();
-        }
-
-        if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN
-                && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
-        if (join.getCondition().getValue() == ConstantExpression.TRUE) {
-            return false;
-        }
-        VariableUtilities.getUsedVariables(join, free);
-
-        AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
-        // see if there's an NTS at the end of the pipeline
-        NestedTupleSourceOperator outerNts = getNts(b0);
-        if (outerNts == null) {
-            AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
-            outerNts = getNts(b1);
-            if (outerNts == null) {
-                return false;
-            }
-        }
-
-        Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
-        if (pkVars == null || pkVars.size() < 1) {
-            // there is no non-trivial primary key, group-by keys are all live variables
-            ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
-            pkVars = new HashSet<LogicalVariable>();
-            VariableUtilities.getLiveVariables(subplanInput, pkVars);
-        }
-        AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
-
-        Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
-        LogicalVariable testForNull = null;
-        AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
-        switch (right.getOperatorTag()) {
-            case UNNEST: {
-                UnnestOperator innerUnnest = (UnnestOperator) right;
-                // Select [ $y != null ]
-                testForNull = innerUnnest.getVariable();
-                break;
-            }
-            case RUNNINGAGGREGATE: {
-                ILogicalOperator inputToRunningAggregate = right.getInputs().get(0).getValue();
-                Set<LogicalVariable> producedVars = new ListSet<LogicalVariable>();
-                VariableUtilities.getProducedVariables(inputToRunningAggregate, producedVars);
-                if (!producedVars.isEmpty()) {
-                    // Select [ $y != null ]
-                    testForNull = producedVars.iterator().next();
-                }
-                break;
-            }
-            case DATASOURCESCAN: {
-                DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
-                // Select [ $y != null ]
-                if (innerScan.getVariables().size() == 1) {
-                    testForNull = innerScan.getVariables().get(0);
-                }
-                break;
-            }
-        }
-        if (testForNull == null) {
-            testForNull = context.newVar();
-            AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(
-                    ConstantExpression.TRUE));
-            tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
-            rightRef.setValue(tmpAsgn);
-            context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
-        }
-
-        IFunctionInfo finfoEq = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.IS_NULL);
-        ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq,
-                new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
-        IFunctionInfo finfoNot = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.NOT);
-        ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot,
-                new MutableObject<ILogicalExpression>(isNullTest));
-        SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest), false,
-                null);
-        GroupByOperator g = new GroupByOperator();
-        Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
-        NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
-        opRef.setValue(g);
-        selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
-
-        List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
-        prodInpList.clear();
-        prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
-
-        ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
-        g.getNestedPlans().add(gPlan);
-        subplanRoot.setValue(op3Ref.getValue());
-        g.getInputs().add(newSubplanRef);
-
-        HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
-        VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
-        underVars.removeAll(pkVars);
-        Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
-        context.updatePrimaryKeys(mappedVars);
-        for (LogicalVariable uv : underVars) {
-            g.getDecorList().add(
-                    new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(
-                            new VariableReferenceExpression(uv))));
-        }
-        OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
-        OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
-        context.computeAndSetTypeEnvironmentForOperator(g);
-        return true;
-    }
-
-    private NestedTupleSourceOperator getNts(AbstractLogicalOperator op) {
-        AbstractLogicalOperator alo = op;
-        do {
-            if (alo.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-                return (NestedTupleSourceOperator) alo;
-            }
-            if (alo.getInputs().size() != 1) {
-                return null;
-            }
-            alo = (AbstractLogicalOperator) alo.getInputs().get(0).getValue();
-        } while (true);
-    }
-
-    protected Set<LogicalVariable> computeGbyVars(AbstractLogicalOperator op, Set<LogicalVariable> freeVars,
-            IOptimizationContext context) throws AlgebricksException {
-        PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
-        List<FunctionalDependency> fdList = context.getFDList(op);
-        if (fdList == null) {
-            return null;
-        }
-        // check if any of the FDs is a key
-        List<LogicalVariable> all = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(op, all);
-        all.retainAll(freeVars);
-        for (FunctionalDependency fd : fdList) {
-            if (fd.getTail().containsAll(all)) {
-                return new HashSet<LogicalVariable>(fd.getHead());
-            }
-        }
-        return null;
-    }
-
-    private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars,
-            IOptimizationContext context, GroupByOperator g,
-            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
-        Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>();
-        for (LogicalVariable ov : vars) {
-            LogicalVariable newVar = context.newVar();
-            ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
-            outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
-                    new MutableObject<ILogicalExpression>(varExpr)));
-            for (ILogicalPlan p : g.getNestedPlans()) {
-                for (Mutable<ILogicalOperator> r : p.getRoots()) {
-                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
-                            context);
-                }
-            }
-            AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue();
-            OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context);
-            m.put(ov, newVar);
-        }
-        return m;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java
deleted file mode 100644
index 3c96bc2..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceProjectsRule.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Projects away unused variables at the earliest possible point.
- * Does a full DFS sweep of the plan adding ProjectOperators in the bottom-up pass.
- * Also, removes projects that have become useless.
- * TODO: This rule 'recklessly' adds as many projects as possible, but there is no guarantee
- * that the overall cost of the plan is reduced since project operators also add a cost.
- */
-public class IntroduceProjectsRule implements IAlgebraicRewriteRule {
-
-    private final Set<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
-    private final Set<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
-    private final Set<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
-    private final List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>();
-    protected boolean hasRun = false;
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        if (hasRun) {
-            return false;
-        }
-        hasRun = true;
-        return introduceProjects(null, -1, opRef, Collections.<LogicalVariable> emptySet(), context);
-    }
-
-    protected boolean introduceProjects(AbstractLogicalOperator parentOp, int parentInputIndex,
-            Mutable<ILogicalOperator> opRef, Set<LogicalVariable> parentUsedVars, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        boolean modified = false;
-
-        usedVars.clear();
-        VariableUtilities.getUsedVariables(op, usedVars);
-
-        // In the top-down pass, maintain a set of variables that are used in op and all its parents.
-        HashSet<LogicalVariable> parentsUsedVars = new HashSet<LogicalVariable>();
-        parentsUsedVars.addAll(parentUsedVars);
-        parentsUsedVars.addAll(usedVars);
-
-        // Descend into children.        
-        for (int i = 0; i < op.getInputs().size(); i++) {
-            Mutable<ILogicalOperator> inputOpRef = op.getInputs().get(i);
-            if (introduceProjects(op, i, inputOpRef, parentsUsedVars, context)) {
-                modified = true;
-            }
-        }
-
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(op);
-        }
-        // In the bottom-up pass, determine which live variables are not used by op's parents.
-        // Such variables are be projected away.
-        liveVars.clear();
-        VariableUtilities.getLiveVariables(op, liveVars);
-        producedVars.clear();
-        VariableUtilities.getProducedVariables(op, producedVars);
-        liveVars.removeAll(producedVars);
-
-        projectVars.clear();
-        for (LogicalVariable liveVar : liveVars) {
-            if (parentsUsedVars.contains(liveVar)) {
-                projectVars.add(liveVar);
-            }
-        }
-
-        // Some of the variables that are live at this op are not used above.
-        if (projectVars.size() != liveVars.size()) {
-            // Add a project operator under each of op's qualifying input branches.
-            for (int i = 0; i < op.getInputs().size(); i++) {
-                ILogicalOperator childOp = op.getInputs().get(i).getValue();
-                liveVars.clear();
-                VariableUtilities.getLiveVariables(childOp, liveVars);
-                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-                vars.addAll(projectVars);
-                // Only retain those variables that are live in the i-th input branch.
-                vars.retainAll(liveVars);
-                if (vars.size() != liveVars.size()) {
-                    ProjectOperator projectOp = new ProjectOperator(vars);
-                    projectOp.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
-                    op.getInputs().get(i).setValue(projectOp);
-                    context.computeAndSetTypeEnvironmentForOperator(projectOp);
-                    modified = true;
-                }
-            }
-        } else if (op.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-            // Check if the existing project has become useless.
-            liveVars.clear();
-            VariableUtilities.getLiveVariables(op.getInputs().get(0).getValue(), liveVars);
-            ProjectOperator projectOp = (ProjectOperator) op;
-            List<LogicalVariable> projectVars = projectOp.getVariables();
-            if (liveVars.size() == projectVars.size() && liveVars.containsAll(projectVars)) {
-                boolean eliminateProject = true;
-                // For UnionAll the variables must also be in exactly the correct order.
-                if (parentOp.getOperatorTag() == LogicalOperatorTag.UNIONALL) {
-                    eliminateProject = canEliminateProjectBelowUnion((UnionAllOperator) parentOp, projectOp,
-                            parentInputIndex);
-                }
-                if (eliminateProject) {
-                    // The existing project has become useless. Remove it.
-                    parentOp.getInputs().get(parentInputIndex).setValue(op.getInputs().get(0).getValue());
-                }
-            }
-        }
-
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(op);
-        }
-        return modified;
-    }
-    
-    private boolean canEliminateProjectBelowUnion(UnionAllOperator unionOp, ProjectOperator projectOp,
-            int unionInputIndex) throws AlgebricksException {
-        List<LogicalVariable> orderedLiveVars = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(projectOp.getInputs().get(0).getValue(), orderedLiveVars);
-        int numVars = orderedLiveVars.size();
-        for (int i = 0; i < numVars; i++) {
-            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varTriple = unionOp.getVariableMappings().get(i);
-            if (unionInputIndex == 0) {
-                if (varTriple.first != orderedLiveVars.get(i)) {
-                    return false;
-                }
-            } else {
-                if (varTriple.second != orderedLiveVars.get(i)) {
-                    return false;
-                }
-            }
-        }
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
deleted file mode 100644
index 71af64b..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class IsolateHyracksOperatorsRule implements IAlgebraicRewriteRule {
-
-    private final PhysicalOperatorTag[] operatorsBelowWhichJobGenIsDisabled;
-
-    public IsolateHyracksOperatorsRule(PhysicalOperatorTag[] operatorsBelowWhichJobGenIsDisabled) {
-        this.operatorsBelowWhichJobGenIsDisabled = operatorsBelowWhichJobGenIsDisabled;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        IPhysicalOperator pt = op.getPhysicalOperator();
-
-        if (pt == null || op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-            return false;
-        }
-        if (!pt.isMicroOperator()) {
-            return testIfExchangeBelow(opRef, context);
-        } else {
-            return testIfExchangeAbove(opRef, context);
-        }
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
-        return false;
-    }
-
-    private boolean testIfExchangeBelow(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        boolean exchInserted = false;
-
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
-            if (c.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
-                if (c.getPhysicalOperator() == null) {
-                    return false;
-                }
-                insertOneToOneExchange(i, context);
-                exchInserted = true;
-            }
-        }
-        IPhysicalOperator pt = op.getPhysicalOperator();
-        if (pt.isJobGenDisabledBelowMe() || arrayContains(operatorsBelowWhichJobGenIsDisabled, pt.getOperatorTag())) {
-            for (Mutable<ILogicalOperator> i : op.getInputs()) {
-                disableJobGenRec(i.getValue());
-            }
-        }
-        return exchInserted;
-    }
-
-    private void disableJobGenRec(ILogicalOperator operator) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) operator;
-        op.disableJobGen();
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            disableJobGenRec(i.getValue());
-        }
-    }
-
-    private boolean testIfExchangeAbove(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-            return false;
-        }
-        boolean exchInserted = false;
-        for (Mutable<ILogicalOperator> i : op.getInputs()) {
-            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
-            IPhysicalOperator cpop = c.getPhysicalOperator();
-            if (c.getOperatorTag() == LogicalOperatorTag.EXCHANGE || cpop == null) {
-                continue;
-            }
-            if (!cpop.isMicroOperator()) {
-                insertOneToOneExchange(i, context);
-                exchInserted = true;
-            }
-        }
-        return exchInserted;
-    }
-
-    private final static <T> boolean arrayContains(T[] array, T tag) {
-        for (int i = 0; i < array.length; i++) {
-            if (array[i] == tag) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private final static void insertOneToOneExchange(Mutable<ILogicalOperator> i, IOptimizationContext context)
-            throws AlgebricksException {
-        ExchangeOperator e = new ExchangeOperator();
-        e.setPhysicalOperator(new OneToOneExchangePOperator());
-        ILogicalOperator inOp = i.getValue();
-
-        e.getInputs().add(new MutableObject<ILogicalOperator>(inOp));
-        i.setValue(e);
-        // e.recomputeSchema();
-        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(e, context);
-        ExecutionMode em = ((AbstractLogicalOperator) inOp).getExecutionMode();
-        e.setExecutionMode(em);
-        e.computeDeliveredPhysicalProperties(context);
-        context.computeAndSetTypeEnvironmentForOperator(e);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java
deleted file mode 100644
index 5867abd..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/MoveFreeVariableOperatorOutOfSubplanRule.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.HashSet;
-import java.util.ListIterator;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.AbstractDecorrelationRule;
-
-/**
- * The rule searches for operators that can be moved outside the subplan.
- *
- * <pre>
- * Before
- * 
- *   %PARENT_PLAN
- *   SUBPLAN{
- *     %NESTED_OPERATORS_B+
- *     ASSIGN || %SUBPLAN
- *     %NESTED_OPERATORS_A*
- *     NESTEDTUPLESOURCE
- *   }
- *   %CHILD_PLAN
- * 
- *   where
- *     %SUBPLAN has one nested plan with a root AGGREGATE operator.
- * 
- * After
- * 
- *   %PARENT_PLAN
- *   SUBPLAN{
- *     %NESTED_OPERATORS_B+
- *     %NESTED_OPERATORS_A*
- *     NESTEDTUPLESOURCE
- *   }
- *   ASSIGN || %SUBPLAN
- *   %CHILD_PLAN
- * </pre>
- */
-public class MoveFreeVariableOperatorOutOfSubplanRule extends AbstractDecorrelationRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
-        if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-            return false;
-        }
-        SubplanOperator subplan = (SubplanOperator) op0;
-
-        Mutable<ILogicalOperator> leftRef = subplan.getInputs().get(0);
-        if (((AbstractLogicalOperator) leftRef.getValue()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-            return false;
-        }
-
-        ListIterator<ILogicalPlan> plansIter = subplan.getNestedPlans().listIterator();
-        ILogicalPlan p = null;
-        while (plansIter.hasNext()) {
-            p = plansIter.next();
-        }
-        if (p == null) {
-            return false;
-        }
-        if (p.getRoots().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> opRef1 = p.getRoots().get(0);
-
-        //The root operator will not be movable. Start with the second op
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getValue();
-        if (op1.getInputs().size() != 1) {
-            return false;
-        }
-        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
-
-        //Get all variables that come from outside of the loop
-        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
-        OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op1, free);
-
-        while (op2Ref != null) {
-            //Get the operator that we want to look at
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-
-            //Make sure we are looking at subplan with a scan/join
-            if (op2.getInputs().size() != 1 || !descOrSelfIsScanOrJoin(op2)) {
-                return false;
-            }
-            boolean notApplicable = false;
-
-            //Get its used variables
-            Set<LogicalVariable> used = new HashSet<LogicalVariable>();
-
-            //not movable if the operator is not an assign or subplan
-            //Might be helpful in the future for other operations in the future
-            if (movableOperator(op2.getOperatorTag())) {
-                if (op2.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-                    VariableUtilities.getUsedVariables(op2, used);
-                } else if (op2.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
-                    // Nested plan must have an aggregate root.
-                    ListIterator<ILogicalPlan> subplansIter = ((SubplanOperator) op2).getNestedPlans().listIterator();
-                    ILogicalPlan plan = null;
-                    while (subplansIter.hasNext()) {
-                        plan = subplansIter.next();
-                    }
-                    if (plan == null) {
-                        return false;
-                    }
-                    if (plan.getRoots().size() != 1) {
-                        return false;
-                    }
-                    ILogicalOperator op3 = plan.getRoots().get(0).getValue();
-                    if (op3.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
-                        return false;
-                    }
-                    // Used variables do not include ones created in the subplan.
-                    VariableUtilities.getUsedVariables(op2, used);
-                    Set<LogicalVariable> subplanProducedAndDown = new HashSet<LogicalVariable>();
-                    VariableUtilities.getProducedVariablesInDescendantsAndSelf(op3, subplanProducedAndDown);
-                    used.removeAll(subplanProducedAndDown);
-                } else {
-                    notApplicable = true;
-                }
-            } else {
-                notApplicable = true;
-            }
-
-            //Make sure that all of its used variables come from outside
-            for (LogicalVariable var : used) {
-                if (!free.contains(var)) {
-                    notApplicable = true;
-                }
-            }
-
-            if (notApplicable) {
-                op2Ref = op2.getInputs().get(0);
-            } else {
-                //Make the input of op2 be the input of op1
-                op2Ref.setValue(op2.getInputs().get(0).getValue());
-
-                //Make the outside of the subplan the input of op2
-                Mutable<ILogicalOperator> outsideRef = op2.getInputs().get(0);
-                outsideRef.setValue(op0.getInputs().get(0).getValue());
-
-                //Make op2 the input of the subplan
-                Mutable<ILogicalOperator> op2OutsideRef = op0.getInputs().get(0);
-                op2OutsideRef.setValue(op2);
-
-                return true;
-            }
-
-        }
-        return false;
-    }
-
-    protected boolean movableOperator(LogicalOperatorTag operatorTag) {
-        return (operatorTag == LogicalOperatorTag.ASSIGN || operatorTag == LogicalOperatorTag.SUBPLAN);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java
deleted file mode 100644
index 6484202..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/NestedSubplanToJoinRule.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * replace Subplan operators with nested loop joins where the join condition is true, if the Subplan
- * does not contain free variables (does not have correlations to the input stream).
- * 
- * @author yingyib
- */
-public class NestedSubplanToJoinRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getValue()))
-            return false;
-        context.addToDontApplySet(this, opRef.getValue());
-
-        ILogicalOperator op1 = opRef.getValue();
-        if (op1.getInputs().size() == 0) {
-            return false;
-        }
-
-        boolean rewritten = false;
-        for (int index = 0; index < op1.getInputs().size(); index++) {
-            AbstractLogicalOperator child = (AbstractLogicalOperator) op1.getInputs().get(index).getValue();
-            if (child.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
-                continue;
-            }
-
-            AbstractOperatorWithNestedPlans subplan = (AbstractOperatorWithNestedPlans) child;
-            Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
-            OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, freeVars);
-            if (!freeVars.isEmpty()) {
-                /**
-                 * the subplan is correlated with the outer plan, other rules can deal with it
-                 */
-                continue;
-            }
-
-            /** get the input operator of the subplan operator */
-            ILogicalOperator subplanInput = subplan.getInputs().get(0).getValue();
-            AbstractLogicalOperator subplanInputOp = (AbstractLogicalOperator) subplanInput;
-
-            /** If the other join branch is a trivial plan, do not do the rewriting. */
-            if (subplanInputOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-                continue;
-            }
-
-            /** get all nested top operators */
-            List<ILogicalPlan> nestedPlans = subplan.getNestedPlans();
-            List<Mutable<ILogicalOperator>> nestedRoots = new ArrayList<Mutable<ILogicalOperator>>();
-            for (ILogicalPlan nestedPlan : nestedPlans) {
-                nestedRoots.addAll(nestedPlan.getRoots());
-            }
-            if (nestedRoots.size() == 0) {
-                /** there is no nested top operators */
-                continue;
-            }
-
-            /** expend the input and roots into a DAG of nested loop joins */
-            Mutable<ILogicalExpression> expr = new MutableObject<ILogicalExpression>(ConstantExpression.TRUE);
-            Mutable<ILogicalOperator> nestedRootRef = nestedRoots.get(0);
-            ILogicalOperator join = new LeftOuterJoinOperator(expr, new MutableObject<ILogicalOperator>(subplanInput),
-                    nestedRootRef);
-
-            /** rewrite the nested tuple source to be empty tuple source */
-            rewriteNestedTupleSource(nestedRootRef);
-
-            for (int i = 1; i < nestedRoots.size(); i++) {
-                join = new LeftOuterJoinOperator(expr, new MutableObject<ILogicalOperator>(join), nestedRoots.get(i));
-            }
-            op1.getInputs().get(index).setValue(join);
-            context.computeAndSetTypeEnvironmentForOperator(join);
-            rewritten = true;
-        }
-        return rewritten;
-    }
-
-    /**
-     * rewrite NestedTupleSource operators to EmptyTupleSource operators
-     * 
-     * @param nestedRootRef
-     */
-    private void rewriteNestedTupleSource(Mutable<ILogicalOperator> nestedRootRef) {
-        AbstractLogicalOperator nestedRoot = (AbstractLogicalOperator) nestedRootRef.getValue();
-        if (nestedRoot.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
-            nestedRootRef.setValue(new EmptyTupleSourceOperator());
-        }
-        List<Mutable<ILogicalOperator>> inputs = nestedRoot.getInputs();
-        for (Mutable<ILogicalOperator> input : inputs) {
-            rewriteNestedTupleSource(input);
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
deleted file mode 100644
index b268e77..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PullSelectOutOfEqJoin implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-
-        if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
-
-        ILogicalExpression expr = join.getCondition().getValue();
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
-        FunctionIdentifier fi = fexp.getFunctionIdentifier();
-        if (!fi.equals(AlgebricksBuiltinFunctions.AND)) {
-            return false;
-        }
-        List<Mutable<ILogicalExpression>> eqVarVarComps = new ArrayList<Mutable<ILogicalExpression>>();
-        List<Mutable<ILogicalExpression>> otherPredicates = new ArrayList<Mutable<ILogicalExpression>>();
-        for (Mutable<ILogicalExpression> arg : fexp.getArguments()) {
-            if (isEqVarVar(arg.getValue())) {
-                eqVarVarComps.add(arg);
-            } else {
-                otherPredicates.add(arg);
-            }
-        }
-        if (eqVarVarComps.isEmpty() || otherPredicates.isEmpty()) {
-            return false;
-        }
-        // pull up
-        ILogicalExpression pulledCond = makeCondition(otherPredicates, context);
-        SelectOperator select = new SelectOperator(new MutableObject<ILogicalExpression>(pulledCond), false, null);
-        ILogicalExpression newJoinCond = makeCondition(eqVarVarComps, context);
-        join.getCondition().setValue(newJoinCond);
-        select.getInputs().add(new MutableObject<ILogicalOperator>(join));
-        opRef.setValue(select);
-        context.computeAndSetTypeEnvironmentForOperator(select);
-        return true;
-    }
-
-    private ILogicalExpression makeCondition(List<Mutable<ILogicalExpression>> predList, IOptimizationContext context) {
-        if (predList.size() > 1) {
-            IFunctionInfo finfo = context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND);
-            return new ScalarFunctionCallExpression(finfo, predList);
-        } else {
-            return predList.get(0).getValue();
-        }
-    }
-
-    private boolean isEqVarVar(ILogicalExpression expr) {
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-        AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
-        if (!f.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.EQ)) {
-            return false;
-        }
-        ILogicalExpression e1 = f.getArguments().get(0).getValue();
-        if (e1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-            return false;
-        } else {
-            ILogicalExpression e2 = f.getArguments().get(1).getValue();
-            return e2.getExpressionTag() == LogicalExpressionTag.VARIABLE;
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java
deleted file mode 100644
index 418ee36..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignBelowUnionAllRule.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Pushes an AssignOperator below a UnionAll operator by creating an new AssignOperator below each of
- * the UnionAllOperator's branches with appropriate variable replacements.
- * This rule can help to enable other rules that are difficult to fire across a UnionAllOperator,
- * for example, eliminating common sub-expressions.
- * Example:
- * Before plan:
- * ...
- * assign [$$20, $$21] <- [funcA($$3), funcB($$6)]
- * union ($$1, $$2, $$3) ($$4, $$5, $$6)
- * union_branch_0
- * ...
- * union_branch_1
- * ...
- * After plan:
- * ...
- * union ($$1, $$2, $$3) ($$4, $$5, $$6) ($$22, $$24, $$20) ($$23, $$25, $$21)
- * assign [$$22, $$23] <- [funcA($$1), funcB($$4)]
- * union_branch_0
- * ...
- * assign [$$24, $$25] <- [funcA($$2), funcB($$5)]
- * union_branch_1
- * ...
- */
-public class PushAssignBelowUnionAllRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (!op.hasInputs()) {
-            return false;
-        }
-
-        boolean modified = false;
-        for (int i = 0; i < op.getInputs().size(); i++) {
-            AbstractLogicalOperator childOp = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
-            if (childOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-                continue;
-            }
-            AssignOperator assignOp = (AssignOperator) childOp;
-            for (Mutable<ILogicalExpression> expr : assignOp.getExpressions()) {
-                if (!expr.getValue().isFunctional()) {
-                    return false;
-                }
-            }
-
-            AbstractLogicalOperator childOfChildOp = (AbstractLogicalOperator) assignOp.getInputs().get(0).getValue();
-            if (childOfChildOp.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
-                continue;
-            }
-            UnionAllOperator unionOp = (UnionAllOperator) childOfChildOp;
-
-            Set<LogicalVariable> assignUsedVars = new HashSet<LogicalVariable>();
-            VariableUtilities.getUsedVariables(assignOp, assignUsedVars);
-
-            List<LogicalVariable> assignVars = assignOp.getVariables();
-
-            AssignOperator[] newAssignOps = new AssignOperator[2];
-            for (int j = 0; j < unionOp.getInputs().size(); j++) {
-                newAssignOps[j] = createAssignBelowUnionAllBranch(unionOp, j, assignOp, assignUsedVars, context);
-            }
-            // Add original assign variables to the union variable mappings.
-            for (int j = 0; j < assignVars.size(); j++) {
-                LogicalVariable first = newAssignOps[0].getVariables().get(j);
-                LogicalVariable second = newAssignOps[1].getVariables().get(j);
-                Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
-                        first, second, assignVars.get(j));
-                unionOp.getVariableMappings().add(varMapping);
-            }
-            context.computeAndSetTypeEnvironmentForOperator(unionOp);
-
-            // Remove original assign operator.
-            op.getInputs().set(i, assignOp.getInputs().get(0));
-            context.computeAndSetTypeEnvironmentForOperator(op);
-            modified = true;
-        }
-
-        return modified;
-    }
-
-    private AssignOperator createAssignBelowUnionAllBranch(UnionAllOperator unionOp, int inputIndex,
-            AssignOperator originalAssignOp, Set<LogicalVariable> assignUsedVars, IOptimizationContext context)
-            throws AlgebricksException {
-        AssignOperator newAssignOp = cloneAssignOperator(originalAssignOp, context);
-        newAssignOp.getInputs()
-                .add(new MutableObject<ILogicalOperator>(unionOp.getInputs().get(inputIndex).getValue()));
-        context.computeAndSetTypeEnvironmentForOperator(newAssignOp);
-        unionOp.getInputs().get(inputIndex).setValue(newAssignOp);
-        int numVarMappings = unionOp.getVariableMappings().size();
-        for (int i = 0; i < numVarMappings; i++) {
-            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varMapping = unionOp.getVariableMappings().get(i);
-            if (assignUsedVars.contains(varMapping.third)) {
-                LogicalVariable replacementVar;
-                if (inputIndex == 0) {
-                    replacementVar = varMapping.first;
-                } else {
-                    replacementVar = varMapping.second;
-                }
-                VariableUtilities.substituteVariables(newAssignOp, varMapping.third, replacementVar, context);
-            }
-        }
-        return newAssignOp;
-    }
-
-    /**
-     * Clones the given assign operator changing the returned variables to be new ones.
-     * Also, leaves the inputs of the clone clear.
-     */
-    private AssignOperator cloneAssignOperator(AssignOperator assignOp, IOptimizationContext context) {
-        List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-        List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
-        int numVars = assignOp.getVariables().size();
-        for (int i = 0; i < numVars; i++) {
-            vars.add(context.newVar());
-            exprs.add(new MutableObject<ILogicalExpression>(assignOp.getExpressions().get(i).getValue()
-                    .cloneExpression()));
-        }
-        AssignOperator assignCloneOp = new AssignOperator(vars, exprs);
-        assignCloneOp.setExecutionMode(assignOp.getExecutionMode());
-        return assignCloneOp;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
deleted file mode 100644
index d8523f6..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class PushAssignDownThroughProductRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-            return false;
-        }
-        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
-            return false;
-        }
-        AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
-        if (join.getCondition().getValue() != ConstantExpression.TRUE) {
-            return false;
-        }
-
-        List<LogicalVariable> used = new ArrayList<LogicalVariable>();
-        VariableUtilities.getUsedVariables(op1, used);
-
-        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
-        ILogicalOperator b0 = b0Ref.getValue();
-        List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(b0, b0Scm);
-        if (b0Scm.containsAll(used)) {
-            // push assign on left branch
-            op2Ref.setValue(b0);
-            b0Ref.setValue(op1);
-            opRef.setValue(op2);
-            return true;
-        } else {
-            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
-            ILogicalOperator b1 = b1Ref.getValue();
-            List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(b1, b1Scm);
-            if (b1Scm.containsAll(used)) {
-                // push assign on right branch
-                op2Ref.setValue(b1);
-                b1Ref.setValue(op1);
-                opRef.setValue(op2);
-                return true;
-            } else {
-                return false;
-            }
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java b/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java
deleted file mode 100644
index 0d24618..0000000
--- a/algebricks/algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushFunctionsBelowJoin.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.algebricks.rewriter.rules;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Pushes function-call expressions below a join if possible.
- * Assigns the result of such function-calls expressions to new variables, and replaces the original
- * expression with a corresponding variable reference expression.
- * This rule can help reduce the cost of computing expensive functions by pushing them below
- * a join (which may blow up the cardinality).
- * Also, this rule may help to enable other rules such as common subexpression elimination, again to reduce
- * the number of calls to expensive functions.
- * 
- * Example: (we are pushing pushMeFunc)
- * 
- * Before plan:
- * assign [$$10] <- [funcA(funcB(pushMeFunc($$3, $$4)))]
- *   join (some condition) 
- *     join_branch_0 where $$3 and $$4 are not live
- *       ...
- *     join_branch_1 where $$3 and $$4 are live
- *       ...
- * 
- * After plan:
- * assign [$$10] <- [funcA(funcB($$11))]
- *   join (some condition) 
- *     join_branch_0 where $$3 and $$4 are not live
- *       ...
- *     join_branch_1 where $$3 and $$4 are live
- *       assign[$$11] <- [pushMeFunc($$3, $$4)]
- *         ...
- */
-public class PushFunctionsBelowJoin implements IAlgebraicRewriteRule {
-
-    private final Set<FunctionIdentifier> toPushFuncIdents;
-    private final List<Mutable<ILogicalExpression>> funcExprs = new ArrayList<Mutable<ILogicalExpression>>();
-    private final List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
-    private final List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
-
-    public PushFunctionsBelowJoin(Set<FunctionIdentifier> toPushFuncIdents) {
-        this.toPushFuncIdents = toPushFuncIdents;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-            return false;
-        }
-        AssignOperator assignOp = (AssignOperator) op;
-
-        // Find a join operator below this assign.
-        Mutable<ILogicalOperator> joinOpRef = findJoinOp(assignOp.getInputs().get(0));
-        if (joinOpRef == null) {
-            return false;
-        }
-        AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinOpRef.getValue();
-
-        // Check if the assign uses a function that we wish to push below the join if possible.
-        funcExprs.clear();
-        gatherFunctionCalls(assignOp, funcExprs);
-        if (funcExprs.isEmpty()) {
-            return false;
-        }
-
-        // Try to push the functions down the input branches of the join.
-        boolean modified = false;
-        if (pushDownFunctions(joinOp, 0, funcExprs, context)) {
-            modified = true;
-        }
-        if (pushDownFunctions(joinOp, 1, funcExprs, context)) {
-            modified = true;
-        }
-        if (modified) {
-            context.computeAndSetTypeEnvironmentForOperator(joinOp);
-        }
-        return modified;
-    }
-
-    private Mutable<ILogicalOperator> findJoinOp(Mutable<ILogicalOperator> opRef) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        switch (op.getOperatorTag()) {
-            case INNERJOIN:
-            case LEFTOUTERJOIN: {
-                return opRef;
-            }
-            // Bail on these operators.
-            case GROUP:
-            case AGGREGATE:
-            case DISTINCT:
-            case UNNEST_MAP: {
-                return null;
-            }
-            // Traverse children.
-            default: {
-                for (Mutable<ILogicalOperator> childOpRef : op.getInputs()) {
-                    return findJoinOp(childOpRef);
-                }
-            }
-        }
-        return null;
-    }
-
-    private void gatherFunctionCalls(AssignOperator assignOp, List<Mutable<ILogicalExpression>> funcExprs) {
-        for (Mutable<ILogicalExpression> exprRef : assignOp.getExpressions()) {
-            gatherFunctionCalls(exprRef, funcExprs);
-        }
-    }
-
-    private void gatherFunctionCalls(Mutable<ILogicalExpression> exprRef, List<Mutable<ILogicalExpression>> funcExprs) {
-        AbstractLogicalExpression expr = (AbstractLogicalExpression) exprRef.getValue();
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return;
-        }
-        // Check whether the function is a function we want to push.
-        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-        if (toPushFuncIdents.contains(funcExpr.getFunctionIdentifier())) {
-            funcExprs.add(exprRef);
-        }
-        // Traverse arguments.
-        for (Mutable<ILogicalExpression> funcArg : funcExpr.getArguments()) {
-            gatherFunctionCalls(funcArg, funcExprs);
-        }
-    }
-
-    private boolean pushDownFunctions(AbstractBinaryJoinOperator joinOp, int inputIndex,
-            List<Mutable<ILogicalExpression>> funcExprs, IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator joinInputOp = joinOp.getInputs().get(inputIndex).getValue();
-        liveVars.clear();
-        VariableUtilities.getLiveVariables(joinInputOp, liveVars);
-        Iterator<Mutable<ILogicalExpression>> funcIter = funcExprs.iterator();
-        List<LogicalVariable> assignVars = null;
-        List<Mutable<ILogicalExpression>> assignExprs = null;
-        while (funcIter.hasNext()) {
-            Mutable<ILogicalExpression> funcExprRef = funcIter.next();
-            ILogicalExpression funcExpr = funcExprRef.getValue();
-            usedVars.clear();
-            funcExpr.getUsedVariables(usedVars);
-            // Check if we can push the function down this branch.
-            if (liveVars.containsAll(usedVars)) {
-                if (assignVars == null) {
-                    assignVars = new ArrayList<LogicalVariable>();
-                    assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
-                }
-                // Replace the original expression with a variable reference expression.
-                LogicalVariable replacementVar = context.newVar();
-                assignVars.add(replacementVar);
-                assignExprs.add(new MutableObject<ILogicalExpression>(funcExpr));
-                funcExprRef.setValue(new VariableReferenceExpression(replacementVar));
-                funcIter.remove();
-            }
-        }
-        // Create new assign operator below the join if any functions can be pushed.
-        if (assignVars != null) {
-            AssignOperator newAssign = new AssignOperator(assignVars, assignExprs);
-            newAssign.getInputs().add(new MutableObject<ILogicalOperator>(joinInputOp));
-            newAssign.setExecutionMode(joinOp.getExecutionMode());
-            joinOp.getInputs().get(inputIndex).setValue(newAssign);
-            context.computeAndSetTypeEnvironmentForOperator(newAssign);
-            return true;
-        }
-        return false;
-    }
-}

[19/51] [partial] incubator-asterixdb-hyracks git commit: Change folder structure for Java repackage

[im...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/utils/WriteValueTools.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/utils/WriteValueTools.java b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/utils/WriteValueTools.java
new file mode 100644
index 0000000..3a1203b
--- /dev/null
+++ b/algebricks/algebricks-data/src/main/java/org/apache/hyracks/algebricks/data/utils/WriteValueTools.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.data.utils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+
+public final class WriteValueTools {
+
+    private final static int[] INT_INTERVALS = { 9, 99, 999, 9999, 99999, 999999, 9999999, 99999999, 999999999,
+            Integer.MAX_VALUE };
+    private final static int[] INT_DIVIDERS = { 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 };
+    private final static int[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
+
+    public static void writeInt(int i, OutputStream os) throws IOException {
+        if (i < 0) {
+            if (i == Integer.MIN_VALUE) {
+                os.write("-2147483648".getBytes());
+                return;
+            }
+            os.write('-');
+            i = -i;
+        }
+        int k = 0;
+        for (; k < INT_INTERVALS.length; k++) {
+            if (i <= INT_INTERVALS[k]) {
+                break;
+            }
+        }
+        while (k > 0) {
+            int q = i / INT_DIVIDERS[k - 1];
+            os.write(DIGITS[q % 10]);
+            k--;
+        }
+        // now, print the units
+        os.write(DIGITS[i % 10]);
+    }
+
+    public static void writeLong(long d, OutputStream os) throws IOException {
+        // now, print the units
+        if (d < 0) {
+            if (d == Long.MIN_VALUE) {
+                os.write("-9223372036854775808".getBytes());
+                return;
+            }
+            os.write('-');
+            d = -d;
+        }
+        long divisor = 1000000000000000000L;
+        while (divisor > d) {
+            divisor = divisor / 10;
+        }
+        while (divisor > 1) {
+            os.write(DIGITS[(int) ((d / divisor) % 10)]);
+            divisor = divisor / 10;
+        }
+        os.write(DIGITS[(int) (d % 10)]);
+    }
+
+    public static void writeUTF8String(byte[] b, int s, int l, OutputStream os) throws IOException {
+        int stringLength = UTF8StringPointable.getUTFLength(b, s);
+        int position = s + 2;
+        int maxPosition = position + stringLength;
+        os.write('\"');
+        while (position < maxPosition) {
+            char c = UTF8StringPointable.charAt(b, position);
+            switch (c) {
+            // escape
+                case '\\':
+                case '"':
+                    os.write('\\');
+                    break;
+            }
+            int sz = UTF8StringPointable.charSize(b, position);
+            while (sz > 0) {
+                os.write(b[position]);
+                position++;
+                sz--;
+            }
+        }
+        os.write('\"');
+    }
+
+    public static void writeUTF8StringNoQuotes(byte[] b, int s, int l, OutputStream os) throws IOException {
+        int stringLength = UTF8StringPointable.getUTFLength(b, s);
+        int position = s + 2;
+        int maxPosition = position + stringLength;
+        while (position < maxPosition) {
+            char c = UTF8StringPointable.charAt(b, position);
+            switch (c) {
+            // escape
+                case '\\':
+                case '"':
+                    os.write('\\');
+                    break;
+            }
+            int sz = UTF8StringPointable.charSize(b, position);
+            while (sz > 0) {
+                os.write(b[position]);
+                position++;
+                sz--;
+            }
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java
deleted file mode 100644
index 1bdf6b2..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public abstract class ASTNode {
-    public enum Tag {
-        ASSIGNMENT,
-        DUMP,
-        LOAD,
-        FILTER,
-
-        SCALAR_FUNCTION,
-        LITERAL,
-        FIELD_ACCESS,
-    }
-
-    public abstract Tag getTag();
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
deleted file mode 100644
index 71cf26d..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public class AssignmentNode extends ASTNode {
-    private String alias;
-
-    private RelationNode relation;
-
-    public AssignmentNode(String alias, RelationNode relation) {
-        this.alias = alias;
-        this.relation = relation;
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.ASSIGNMENT;
-    }
-
-    public String getAlias() {
-        return alias;
-    }
-
-    public RelationNode getRelation() {
-        return relation;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java
deleted file mode 100644
index 42fca63..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public class DumpNode extends RelationNode {
-    private final String file;
-    private final String alias;
-
-    public DumpNode(String file, String alias) {
-        this.file = file;
-        this.alias = alias;
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.DUMP;
-    }
-
-    public String getFile() {
-        return file;
-    }
-
-    public String getAlias() {
-        return alias;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
deleted file mode 100644
index c1482c2..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public abstract class ExpressionNode extends ASTNode {
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
deleted file mode 100644
index 90b54e6..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public class FieldAccessExpressionNode extends ExpressionNode {
-    private String relationName;
-
-    private String fieldName;
-
-    public FieldAccessExpressionNode(String relationName, String fieldName) {
-        this.relationName = relationName;
-        this.fieldName = fieldName;
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.FIELD_ACCESS;
-    }
-
-    public String getRelationName() {
-        return relationName;
-    }
-
-    public String getFieldName() {
-        return fieldName;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java
deleted file mode 100644
index b917cbb..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public class FilterNode extends RelationNode {
-    private String alias;
-
-    private ExpressionNode expression;
-
-    public FilterNode(String alias, ExpressionNode expression) {
-        this.alias = alias;
-        this.expression = expression;
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.FILTER;
-    }
-
-    public String getAlias() {
-        return alias;
-    }
-
-    public ExpressionNode getExpression() {
-        return expression;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
deleted file mode 100644
index 523efb1..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public enum FunctionTag {
-    BOOLEAN_AND,
-    BOOLEAN_OR,
-    BOOLEAN_NOT,
-
-    EQ,
-    NEQ,
-    LT,
-    LTE,
-    GT,
-    GTE,
-
-    ADD,
-    SUBTRACT,
-    MULTIPLY,
-    DIVIDE,
-    MOD,
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
deleted file mode 100644
index 7b3f2bd..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-
-public class LiteralExpressionNode extends ExpressionNode {
-    private String image;
-
-    private Type type;
-
-    public LiteralExpressionNode(String image, Type type) {
-        this.image = image;
-        this.type = type;
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.LITERAL;
-    }
-
-    public String getImage() {
-        return image;
-    }
-
-    public Type getType() {
-        return type;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java
deleted file mode 100644
index 0ff16b9..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
-
-public class LoadNode extends RelationNode {
-    private String dataFile;
-
-    private Schema schema;
-
-    public LoadNode(String dataFile, Schema schema) {
-        this.dataFile = dataFile;
-        this.schema = schema;
-    }
-    
-    @Override
-    public Tag getTag() {
-        return Tag.LOAD;
-    }
-
-    public String getDataFile() {
-        return dataFile;
-    }
-
-    public Schema getSchema() {
-        return schema;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java
deleted file mode 100644
index 9bd7324..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-public abstract class RelationNode extends ASTNode {
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
deleted file mode 100644
index a0ce89b..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
-
-import java.util.List;
-
-public class ScalarFunctionExpressionNode extends ExpressionNode {
-    private FunctionTag fTag;
-
-    private String fName;
-
-    private List<ASTNode> arguments;
-
-    public ScalarFunctionExpressionNode(FunctionTag fTag, String fName, List<ASTNode> arguments) {
-        this.fTag = fTag;
-        this.fName = fName;
-        this.arguments = arguments;
-    }
-
-    @Override
-    public Tag getTag() {
-        return Tag.SCALAR_FUNCTION;
-    }
-
-    public FunctionTag getFunctionTag() {
-        return fTag;
-    }
-
-    public String getFunctionName() {
-        return fName;
-    }
-
-    public List<ASTNode> getArguments() {
-        return arguments;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
deleted file mode 100644
index 82276d0..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-
-public final class ConstantValue implements IAlgebricksConstantValue {
-    private final Type type;
-
-    private final String image;
-
-    public ConstantValue(Type type, String image) {
-        this.type = type;
-        this.image = image;
-    }
-
-    public Type getType() {
-        return type;
-    }
-
-    public String getImage() {
-        return image;
-    }
-
-    @Override
-    public boolean isFalse() {
-        return false;
-    }
-
-    @Override
-    public boolean isNull() {
-        return false;
-    }
-
-    @Override
-    public boolean isTrue() {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
deleted file mode 100644
index 820d016..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
-
-import java.io.Reader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
-import edu.uci.ics.hyracks.algebricks.compiler.api.ICompiler;
-import edu.uci.ics.hyracks.algebricks.compiler.api.ICompilerFactory;
-import edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController;
-import edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.LogicalExpressionJobGenToExpressionRuntimeProviderAdapter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
-import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.data.ISerializerDeserializerProvider;
-import edu.uci.ics.hyracks.algebricks.data.ITypeTraitProvider;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.AssignmentNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.DumpNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ExpressionNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FieldAccessExpressionNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FilterNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FunctionTag;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.LiteralExpressionNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.LoadNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.RelationNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ScalarFunctionExpressionNode;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletFileDataSink;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletFileDataSource;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.parser.ParseException;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.parser.PigletParser;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.rewriter.PigletRewriteRuleset;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.PigletExpressionJobGen;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-
-public class PigletCompiler {
-    private static final Logger LOGGER = Logger.getLogger(PigletCompiler.class.getName());
-
-    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultLogicalRewrites() {
-        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
-        SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
-        SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
-        SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
-                PigletRewriteRuleset.buildTypeInferenceRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlFullDfs,
-                PigletRewriteRuleset.buildNormalizationRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
-                PigletRewriteRuleset.buildCondPushDownRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
-                PigletRewriteRuleset.buildJoinInferenceRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
-                PigletRewriteRuleset.buildOpPushDownRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
-                PigletRewriteRuleset.buildDataExchangeRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
-                PigletRewriteRuleset.buildConsolidationRuleCollection()));
-        return defaultLogicalRewrites;
-    }
-
-    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultPhysicalRewrites() {
-        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
-        SequentialOnceRuleController seqOnceCtrlAllLevels = new SequentialOnceRuleController(true);
-        SequentialOnceRuleController seqOnceCtrlTopLevel = new SequentialOnceRuleController(false);
-        defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels,
-                PigletRewriteRuleset.buildPhysicalRewritesAllLevelsRuleCollection()));
-        defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlTopLevel,
-                PigletRewriteRuleset.buildPhysicalRewritesTopLevelRuleCollection()));
-        defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels,
-                PigletRewriteRuleset.prepareForJobGenRuleCollection()));
-        return defaultPhysicalRewrites;
-    }
-
-    private final ICompilerFactory cFactory;
-
-    private final PigletMetadataProvider metadataProvider;
-
-    private int varCounter;
-
-    private ILogicalOperator previousOp;
-
-    public PigletCompiler() {
-        HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder();
-        builder.setLogicalRewrites(buildDefaultLogicalRewrites());
-        builder.setPhysicalRewrites(buildDefaultPhysicalRewrites());
-        builder.setSerializerDeserializerProvider(new ISerializerDeserializerProvider() {
-            @SuppressWarnings("unchecked")
-            @Override
-            public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException {
-                return null;
-            }
-        });
-        builder.setTypeTraitProvider(new ITypeTraitProvider() {
-            public ITypeTraits getTypeTrait(Object type) {
-                return null;
-            }
-        });
-        builder.setPrinterProvider(PigletPrinterFactoryProvider.INSTANCE);
-        builder.setExpressionRuntimeProvider(new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter(
-                new PigletExpressionJobGen()));
-        builder.setExpressionTypeComputer(new IExpressionTypeComputer() {
-            @Override
-            public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
-                    IVariableTypeEnvironment env) throws AlgebricksException {
-                return null;
-            }
-        });
-        cFactory = builder.create();
-        metadataProvider = new PigletMetadataProvider();
-    }
-
-    public List<ASTNode> parse(Reader in) throws ParseException {
-        PigletParser parser = new PigletParser(in);
-        List<ASTNode> statements = parser.Statements();
-        return statements;
-    }
-
-    public JobSpecification compile(List<ASTNode> ast) throws AlgebricksException, PigletException {
-        ILogicalPlan plan = translate(ast);
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Translated Plan:");
-            LOGGER.info(getPrettyPrintedPlan(plan));
-        }
-        ICompiler compiler = cFactory.createCompiler(plan, metadataProvider, varCounter);
-        compiler.optimize();
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Optimized Plan:");
-            LOGGER.info(getPrettyPrintedPlan(plan));
-        }
-        return compiler.createJob(null, null);
-    }
-
-    private ILogicalPlan translate(List<ASTNode> ast) throws PigletException {
-        Map<String, Relation> symMap = new HashMap<String, Relation>();
-        List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
-        previousOp = null;
-        for (ASTNode an : ast) {
-            switch (an.getTag()) {
-                case DUMP: {
-                    DumpNode dn = (DumpNode) an;
-                    Relation input = symMap.get(dn.getAlias());
-                    List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
-                    for (LogicalVariable v : input.schema.values()) {
-                        expressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v)));
-                    }
-                    PigletFileDataSink dataSink = new PigletFileDataSink(dn.getFile());
-                    ILogicalOperator op = new WriteOperator(expressions, dataSink);
-                    op.getInputs().add(new MutableObject<ILogicalOperator>(input.op));
-                    roots.add(new MutableObject<ILogicalOperator>(op));
-                }
-                    break;
-
-                case ASSIGNMENT: {
-                    AssignmentNode asn = (AssignmentNode) an;
-                    String alias = asn.getAlias();
-                    RelationNode rn = asn.getRelation();
-                    Relation rel = translate(rn, symMap);
-                    previousOp = rel.op;
-                    rel.alias = alias;
-                    symMap.put(alias, rel);
-                }
-                    break;
-            }
-        }
-        return new ALogicalPlanImpl(roots);
-    }
-
-    private Relation translate(RelationNode rn, Map<String, Relation> symMap) throws PigletException {
-        switch (rn.getTag()) {
-            case LOAD: {
-                LoadNode ln = (LoadNode) rn;
-                String file = ln.getDataFile();
-                Schema schema = ln.getSchema();
-                List<Pair<String, Type>> fieldsSchema = schema.getSchema();
-                List<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-                List<Object> types = new ArrayList<Object>();
-                Relation rel = new Relation();
-                for (Pair<String, Type> p : fieldsSchema) {
-                    LogicalVariable v = newVariable();
-                    rel.schema.put(p.first, v);
-                    variables.add(v);
-                    types.add(p.second);
-                }
-                PigletFileDataSource ds = new PigletFileDataSource(file, types.toArray());
-                rel.op = new DataSourceScanOperator(variables, ds);
-                rel.op.getInputs().add(
-                        new MutableObject<ILogicalOperator>(previousOp == null ? new EmptyTupleSourceOperator()
-                                : previousOp));
-                return rel;
-            }
-
-            case FILTER: {
-                FilterNode fn = (FilterNode) rn;
-                String alias = fn.getAlias();
-                ExpressionNode conditionNode = fn.getExpression();
-                Relation inputRel = findInputRelation(alias, symMap);
-                Pair<Relation, LogicalVariable> tempInput = translateScalarExpression(inputRel, conditionNode);
-                Relation rel = new Relation();
-                rel.op = new SelectOperator(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
-                        tempInput.second)), false, null);
-                rel.op.getInputs().add(new MutableObject<ILogicalOperator>(tempInput.first.op));
-                rel.schema.putAll(tempInput.first.schema);
-                return rel;
-            }
-        }
-        throw new IllegalArgumentException("Unknown node: " + rn.getTag() + " encountered");
-    }
-
-    private Pair<Relation, LogicalVariable> translateScalarExpression(Relation inputRel, ExpressionNode expressionNode)
-            throws PigletException {
-        switch (expressionNode.getTag()) {
-            case FIELD_ACCESS: {
-                FieldAccessExpressionNode faen = (FieldAccessExpressionNode) expressionNode;
-                String fieldName = faen.getFieldName();
-                LogicalVariable lVar = findField(fieldName, inputRel.schema);
-                return new Pair<Relation, LogicalVariable>(inputRel, lVar);
-            }
-
-            case LITERAL: {
-                LiteralExpressionNode len = (LiteralExpressionNode) expressionNode;
-                String image = len.getImage();
-                Type type = len.getType();
-                ConstantExpression ce = new ConstantExpression(new ConstantValue(type, image));
-                Relation rel = new Relation();
-                LogicalVariable var = newVariable();
-                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-                vars.add(var);
-
-                List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
-                exprs.add(new MutableObject<ILogicalExpression>(ce));
-
-                rel.op = new AssignOperator(vars, exprs);
-                rel.op.getInputs().add(new MutableObject<ILogicalOperator>(inputRel.op));
-                rel.schema.putAll(inputRel.schema);
-
-                return new Pair<Relation, LogicalVariable>(rel, var);
-            }
-
-            case SCALAR_FUNCTION: {
-                ScalarFunctionExpressionNode sfen = (ScalarFunctionExpressionNode) expressionNode;
-                List<Mutable<ILogicalExpression>> argExprs = new ArrayList<Mutable<ILogicalExpression>>();
-                List<ASTNode> arguments = sfen.getArguments();
-                Relation rel = inputRel;
-                for (ASTNode a : arguments) {
-                    Pair<Relation, LogicalVariable> argPair = translateScalarExpression(rel, (ExpressionNode) a);
-                    rel = argPair.first;
-                    argExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(argPair.second)));
-                }
-                Relation outRel = new Relation();
-                outRel.schema.putAll(rel.schema);
-                LogicalVariable var = newVariable();
-                List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-                vars.add(var);
-
-                IFunctionInfo fInfo = lookupFunction(sfen.getFunctionTag(), sfen.getFunctionName());
-
-                List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
-                exprs.add(new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(fInfo, argExprs)));
-                outRel.op = new AssignOperator(vars, exprs);
-                outRel.op.getInputs().add(new MutableObject<ILogicalOperator>(rel.op));
-                return new Pair<Relation, LogicalVariable>(outRel, var);
-            }
-        }
-        return null;
-    }
-
-    private IFunctionInfo lookupFunction(FunctionTag functionTag, String functionName) throws PigletException {
-        switch (functionTag) {
-            case EQ:
-                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.EQ);
-
-            case NEQ:
-                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.NEQ);
-
-            case LT:
-                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.LT);
-
-            case LTE:
-                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.LE);
-
-            case GT:
-                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.GT);
-
-            case GTE:
-                return metadataProvider.lookupFunction(AlgebricksBuiltinFunctions.GE);
-        }
-        throw new PigletException("Unsupported function: " + functionTag);
-    }
-
-    private LogicalVariable newVariable() {
-        return new LogicalVariable(varCounter++);
-    }
-
-    private LogicalVariable findField(String fieldName, Map<String, LogicalVariable> schema) throws PigletException {
-        LogicalVariable var = schema.get(fieldName);
-        if (var == null) {
-            throw new PigletException("Unable to find field named: " + fieldName);
-        }
-        return var;
-    }
-
-    private Relation findInputRelation(String alias, Map<String, Relation> symMap) throws PigletException {
-        Relation rel = symMap.get(alias);
-        if (rel == null) {
-            throw new PigletException("Unknown alias " + alias + "referenced");
-        }
-        return rel;
-    }
-
-    private static class Relation {
-        String alias;
-        ILogicalOperator op;
-        final Map<String, LogicalVariable> schema;
-
-        public Relation() {
-            schema = new LinkedHashMap<String, LogicalVariable>();
-        }
-    }
-
-    private String getPrettyPrintedPlan(ILogicalPlan plan) throws AlgebricksException {
-        LogicalOperatorPrettyPrintVisitor v = new LogicalOperatorPrettyPrintVisitor();
-        StringBuilder buffer = new StringBuilder();
-        PlanPrettyPrinter.printPlan(plan, buffer, v, 0);
-        return buffer.toString();
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
deleted file mode 100644
index beb4e32..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
-
-import java.io.IOException;
-import java.io.PrintStream;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.data.IPrinter;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
-import edu.uci.ics.hyracks.algebricks.data.impl.IntegerPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.data.utils.WriteValueTools;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-import edu.uci.ics.hyracks.data.std.primitive.FloatPointable;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-
-public class PigletPrinterFactoryProvider implements IPrinterFactoryProvider {
-
-    public static final PigletPrinterFactoryProvider INSTANCE = new PigletPrinterFactoryProvider();
-
-    private PigletPrinterFactoryProvider() {
-    }
-
-    @Override
-    public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException {
-        Type t = (Type) type;
-        switch (t.getTag()) {
-            case INTEGER:
-                return IntegerPrinterFactory.INSTANCE;
-            case CHAR_ARRAY:
-                return CharArrayPrinterFactory.INSTANCE;
-            case FLOAT:
-                return FloatPrinterFactory.INSTANCE;
-            default:
-                throw new UnsupportedOperationException();
-
-        }
-    }
-
-    public static class CharArrayPrinterFactory implements IPrinterFactory {
-
-        private static final long serialVersionUID = 1L;
-
-        public static final CharArrayPrinterFactory INSTANCE = new CharArrayPrinterFactory();
-
-        private CharArrayPrinterFactory() {
-        }
-
-        @Override
-        public IPrinter createPrinter() {
-            return new IPrinter() {
-                @Override
-                public void init() throws AlgebricksException {
-                }
-
-                @Override
-                public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
-                    try {
-                        WriteValueTools.writeUTF8String(b, s, l, ps);
-                    } catch (IOException e) {
-                        throw new AlgebricksException(e);
-                    }
-                }
-            };
-        }
-    }
-
-    public static class FloatPrinterFactory implements IPrinterFactory {
-
-        private static final long serialVersionUID = 1L;
-
-        public static final FloatPrinterFactory INSTANCE = new FloatPrinterFactory();
-
-        private FloatPrinterFactory() {
-        }
-
-        @Override
-        public IPrinter createPrinter() {
-            return new IPrinter() {
-                @Override
-                public void init() throws AlgebricksException {
-                }
-
-                @Override
-                public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
-                    ps.print(FloatPointable.getFloat(b, s));
-                }
-            };
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
deleted file mode 100644
index aa28f34..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions;
-
-public class PigletException extends Exception {
-    private static final long serialVersionUID = 1L;
-
-    public PigletException(String message) {
-        super(message);
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
deleted file mode 100644
index 1ac4259..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
-
-import java.io.File;
-
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-
-public class FileSplitUtils {
-    public static FileSplit[] parseFileSplits(String fileSplits) {
-        String[] splits = fileSplits.split(",");
-        FileSplit[] fSplits = new FileSplit[splits.length];
-        for (int i = 0; i < splits.length; ++i) {
-            String s = splits[i].trim();
-            int idx = s.indexOf(':');
-            if (idx < 0) {
-                throw new IllegalArgumentException("File split " + s + " not well formed");
-            }
-            fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1))));
-        }
-        return fSplits;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
deleted file mode 100644
index 6ab624d..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-
-public class PigletFileDataSink implements IDataSink {
-    private String file;
-
-    private FileSplit[] fileSplits;
-
-    private IPartitioningProperty partProp;
-
-    public PigletFileDataSink(String file) {
-        this.file = file;
-        fileSplits = FileSplitUtils.parseFileSplits(file);
-        partProp = new RandomPartitioningProperty(new FileSplitDomain(fileSplits));
-    }
-
-    @Override
-    public Object getId() {
-        return file;
-    }
-
-    public FileSplit[] getFileSplits() {
-        return fileSplits;
-    }
-
-    @Override
-    public Object[] getSchemaTypes() {
-        return null;
-    }
-
-    @Override
-    public IPartitioningProperty getPartitioningProperty() {
-        return partProp;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
deleted file mode 100644
index 076e15a..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-
-public class PigletFileDataSource implements IDataSource<String> {
-    private final String file;
-
-    private final Object[] types;
-
-    private final FileSplit[] fileSplits;
-
-    private IDataSourcePropertiesProvider propProvider;
-
-    public PigletFileDataSource(String file, Object[] types) {
-        this.file = file;
-        this.types = types;
-        fileSplits = FileSplitUtils.parseFileSplits(file);
-        final IPhysicalPropertiesVector vec = new StructuralPropertiesVector(new RandomPartitioningProperty(
-                new FileSplitDomain(fileSplits)), new ArrayList<ILocalStructuralProperty>());
-        propProvider = new IDataSourcePropertiesProvider() {
-            @Override
-            public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
-                return vec;
-            }
-        };
-    }
-
-    @Override
-    public String getId() {
-        return file;
-    }
-
-    @Override
-    public Object[] getSchemaTypes() {
-        return types;
-    }
-
-    public FileSplit[] getFileSplits() {
-        return fileSplits;
-    }
-
-    @Override
-    public IDataSourcePropertiesProvider getPropertiesProvider() {
-        return propProvider;
-    }
-
-    @Override
-    public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList) {
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
deleted file mode 100644
index ba5c5aa..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFunction.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
-
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AbstractFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-
-public class PigletFunction extends AbstractFunctionInfo {
-    private static final long serialVersionUID = 1L;
-
-    private final FunctionIdentifier fid;
-
-    public PigletFunction(FunctionIdentifier fid) {
-        super(true);
-        this.fid = fid;
-    }
-
-    @Override
-    public FunctionIdentifier getFunctionIdentifier() {
-        return fid;
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
deleted file mode 100644
index 099520d..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.JobGenContext;
-import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
-import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
-import edu.uci.ics.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.operators.std.SinkWriterRuntimeFactory;
-import edu.uci.ics.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
-import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
-import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
-
-public class PigletMetadataProvider implements IMetadataProvider<String, String> {
-    private static final Map<FunctionIdentifier, PigletFunction> FN_MAP;
-
-    static {
-        Map<FunctionIdentifier, PigletFunction> map = new HashMap<FunctionIdentifier, PigletFunction>();
-
-        map.put(AlgebricksBuiltinFunctions.EQ, new PigletFunction(AlgebricksBuiltinFunctions.EQ));
-
-        FN_MAP = Collections.unmodifiableMap(map);
-    }
-
-    @Override
-    public IDataSource<String> findDataSource(String id) throws AlgebricksException {
-        return null;
-    }
-
-    @SuppressWarnings("unchecked")
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource,
-            List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
-            List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
-            IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
-            throws AlgebricksException {
-        PigletFileDataSource ds = (PigletFileDataSource) dataSource;
-
-        FileSplit[] fileSplits = ds.getFileSplits();
-        String[] locations = new String[fileSplits.length];
-        for (int i = 0; i < fileSplits.length; ++i) {
-            locations[i] = fileSplits[i].getNodeName();
-        }
-        IFileSplitProvider fsp = new ConstantFileSplitProvider(fileSplits);
-
-        Object[] colTypes = ds.getSchemaTypes();
-        IValueParserFactory[] vpfs = new IValueParserFactory[colTypes.length];
-        ISerializerDeserializer[] serDesers = new ISerializerDeserializer[colTypes.length];
-
-        for (int i = 0; i < colTypes.length; ++i) {
-            Type colType = (Type) colTypes[i];
-            IValueParserFactory vpf;
-            ISerializerDeserializer serDeser;
-            switch (colType.getTag()) {
-                case INTEGER:
-                    vpf = IntegerParserFactory.INSTANCE;
-                    serDeser = IntegerSerializerDeserializer.INSTANCE;
-                    break;
-
-                case CHAR_ARRAY:
-                    vpf = UTF8StringParserFactory.INSTANCE;
-                    serDeser = UTF8StringSerializerDeserializer.INSTANCE;
-                    break;
-
-                case FLOAT:
-                    vpf = FloatParserFactory.INSTANCE;
-                    serDeser = FloatSerializerDeserializer.INSTANCE;
-                    break;
-
-                default:
-                    throw new UnsupportedOperationException();
-            }
-            vpfs[i] = vpf;
-            serDesers[i] = serDeser;
-        }
-
-        ITupleParserFactory tpf = new DelimitedDataTupleParserFactory(vpfs, ',');
-        RecordDescriptor rDesc = new RecordDescriptor(serDesers);
-
-        IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
-        AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
-        return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
-    }
-
-    @Override
-    public boolean scannerOperatorIsLeaf(IDataSource<String> dataSource) {
-        return true;
-    }
-
-    @Override
-    public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
-            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
-            throws AlgebricksException {
-        PigletFileDataSink ds = (PigletFileDataSink) sink;
-        FileSplit[] fileSplits = ds.getFileSplits();
-        String[] locations = new String[fileSplits.length];
-        for (int i = 0; i < fileSplits.length; ++i) {
-            locations[i] = fileSplits[i].getNodeName();
-        }
-        IPushRuntimeFactory prf = new SinkWriterRuntimeFactory(printColumns, printerFactories, fileSplits[0]
-                .getLocalFile().getFile(), PrinterBasedWriterFactory.INSTANCE, inputDesc);
-        AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
-        return new Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint>(prf, constraint);
-    }
-
-    @Override
-    public IDataSourceIndex<String, String> findDataSourceIndex(String indexId, String dataSourceId)
-            throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getResultHandleRuntime(IDataSink sink,
-            int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc, boolean ordered,
-            JobSpecification spec) throws AlgebricksException {
-        return null;
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(
-            IDataSource<String> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> keys,
-            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, JobGenContext context,
-            JobSpecification jobSpec) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
-            IDataSourceIndex<String, String> dataSource,
-            IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
-            IVariableTypeEnvironment typeEnv,
-            List<LogicalVariable> primaryKeys,
-            List<LogicalVariable> secondaryKeys,
-            List<LogicalVariable> additionalNonKeyFields,
-            ILogicalExpression filterExpr,
-            RecordDescriptor recordDesc, JobGenContext context,
-            JobSpecification spec, boolean bulkload) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
-            IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
-            IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys,
-            List<LogicalVariable> secondaryKeys, List<LogicalVariable> additionalNonKeyFields,
-            ILogicalExpression filterExpr, RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec)
-            throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getTokenizerRuntime(
-            IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas,
-            IVariableTypeEnvironment typeEnv, List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
-            ILogicalExpression filterExpr, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification spec, boolean bulkload) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public IFunctionInfo lookupFunction(FunctionIdentifier fid) {
-        return FN_MAP.get(fid);
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
-            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
-            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification jobSpec, boolean bulkload) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-    @Override
-    public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<String> dataSource,
-            IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
-            LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor recordDesc,
-            JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
-        // TODO Auto-generated method stub
-        return null;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/9939b48e/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
----------------------------------------------------------------------
diff --git a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java b/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
deleted file mode 100644
index 1c2cfae..0000000
--- a/algebricks/algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.examples.piglet.rewriter;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.BreakSelectIntoConjunctsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.ComplexJoinInferenceRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.ConsolidateAssignsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.ConsolidateSelectsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.EliminateSubplanRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractGbyExpressionsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecorVarsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.InferTypesRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.InlineVariablesRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.IsolateHyracksOperatorsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.PullSelectOutOfEqJoin;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.CopyLimitDownRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushProjectDownRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushProjectIntoDataSourceScanRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushSelectDownRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushSelectIntoJoinRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.ReinferAllTypesRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.RemoveUnusedAssignAndAggregateRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetAlgebricksPhysicalOperatorsRule;
-import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetExecutionModeRule;
-
-public class PigletRewriteRuleset {
-
-    public final static List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() {
-        List<IAlgebraicRewriteRule> typeInfer = new LinkedList<IAlgebraicRewriteRule>();
-        typeInfer.add(new InferTypesRule());
-        return typeInfer;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() {
-        List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>();
-        normalization.add(new EliminateSubplanRule());
-        // TODO: This rule is incorrect and has been removed. Its replacement in
-        // Asterix (PushAggFuncIntoStandaloneAggregateRule)
-        // is language-specific.
-        // normalization.add(new IntroduceGroupByForStandaloneAggregRule());
-        normalization.add(new BreakSelectIntoConjunctsRule());
-        normalization.add(new PushSelectIntoJoinRule());
-        normalization.add(new ExtractGbyExpressionsRule());
-        return normalization;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildCondPushDownRuleCollection() {
-        List<IAlgebraicRewriteRule> condPushDown = new LinkedList<IAlgebraicRewriteRule>();
-        condPushDown.add(new PushSelectDownRule());
-        condPushDown.add(new InlineVariablesRule());
-        condPushDown.add(new FactorRedundantGroupAndDecorVarsRule());
-        condPushDown.add(new EliminateSubplanRule());
-        return condPushDown;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildJoinInferenceRuleCollection() {
-        List<IAlgebraicRewriteRule> joinInference = new LinkedList<IAlgebraicRewriteRule>();
-        joinInference.add(new InlineVariablesRule());
-        joinInference.add(new ComplexJoinInferenceRule());
-        return joinInference;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildOpPushDownRuleCollection() {
-        List<IAlgebraicRewriteRule> opPushDown = new LinkedList<IAlgebraicRewriteRule>();
-        opPushDown.add(new PushProjectDownRule());
-        opPushDown.add(new PushSelectDownRule());
-        return opPushDown;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() {
-        List<IAlgebraicRewriteRule> dataExchange = new LinkedList<IAlgebraicRewriteRule>();
-        dataExchange.add(new SetExecutionModeRule());
-        return dataExchange;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() {
-        List<IAlgebraicRewriteRule> consolidation = new LinkedList<IAlgebraicRewriteRule>();
-        consolidation.add(new ConsolidateSelectsRule());
-        consolidation.add(new ConsolidateAssignsRule());
-        consolidation.add(new RemoveUnusedAssignAndAggregateRule());
-        return consolidation;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() {
-        List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>();
-        physicalPlanRewrites.add(new PullSelectOutOfEqJoin());
-        physicalPlanRewrites.add(new SetAlgebricksPhysicalOperatorsRule());
-        physicalPlanRewrites.add(new EnforceStructuralPropertiesRule());
-        physicalPlanRewrites.add(new PushProjectDownRule());
-        physicalPlanRewrites.add(new CopyLimitDownRule());
-        return physicalPlanRewrites;
-    }
-
-    public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection() {
-        List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>();
-        physicalPlanRewrites.add(new CopyLimitDownRule());
-        return physicalPlanRewrites;
-    }
-
-    public final static List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() {
-        List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<IAlgebraicRewriteRule>();
-        prepareForJobGenRewrites.add(new IsolateHyracksOperatorsRule(
-                HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
-        prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule());
-        // Re-infer all types, so that, e.g., the effect of not-is-null is
-        // propagated.
-        prepareForJobGenRewrites.add(new PushProjectIntoDataSourceScanRule());
-        prepareForJobGenRewrites.add(new ReinferAllTypesRule());
-        return prepareForJobGenRewrites;
-    }
-
-}
\ No newline at end of file