You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/04/24 20:45:48 UTC
[21/85] [abbrv] [partial] incubator-asterixdb-hyracks git commit:
Move Pregelix and Hivesterix codebase to new repositories: 1. Move Pregelix
codebase to https://github.com/pregelix/pregelix;
2. Move Hivesterix codebase to https://code.google.com/p/hives
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java
deleted file mode 100644
index a4022d1..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/JoinVisitor.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
-import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-
-@SuppressWarnings("rawtypes")
-public class JoinVisitor extends DefaultVisitor {
-
- /**
- * reduce sink operator to variables
- */
- private HashMap<Operator, List<LogicalVariable>> reduceSinkToKeyVariables = new HashMap<Operator, List<LogicalVariable>>();
-
- /**
- * reduce sink operator to variables
- */
- private HashMap<Operator, List<String>> reduceSinkToFieldNames = new HashMap<Operator, List<String>>();
-
- /**
- * reduce sink operator to variables
- */
- private HashMap<Operator, List<TypeInfo>> reduceSinkToTypes = new HashMap<Operator, List<TypeInfo>>();
-
- /**
- * map a join operator (in hive) to its parent operators (in hive)
- */
- private HashMap<Operator, List<Operator>> operatorToHiveParents = new HashMap<Operator, List<Operator>>();
-
- /**
- * map a join operator (in hive) to its parent operators (in asterix)
- */
- private HashMap<Operator, List<ILogicalOperator>> operatorToAsterixParents = new HashMap<Operator, List<ILogicalOperator>>();
-
- /**
- * the latest traversed reduce sink operator
- */
- private Operator latestReduceSink = null;
-
- /**
- * the latest generated parent for join
- */
- private ILogicalOperator latestAlgebricksOperator = null;
-
- /**
- * process a join operator
- */
- @Override
- public Mutable<ILogicalOperator> visit(JoinOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
- Translator t) {
- latestAlgebricksOperator = AlgebricksParentOperator.getValue();
- translateJoinOperatorPreprocess(operator, t);
- List<Operator> parents = operatorToHiveParents.get(operator);
- if (parents.size() < operator.getParentOperators().size()) {
- return null;
- } else {
- ILogicalOperator joinOp = translateJoinOperator(operator, AlgebricksParentOperator, t);
- // clearStatus();
- return new MutableObject<ILogicalOperator>(joinOp);
- }
- }
-
- private void reorder(Byte[] order, List<ILogicalOperator> parents, List<Operator> hiveParents) {
- ILogicalOperator[] lops = new ILogicalOperator[parents.size()];
- Operator[] ops = new Operator[hiveParents.size()];
-
- for (Operator op : hiveParents) {
- ReduceSinkOperator rop = (ReduceSinkOperator) op;
- ReduceSinkDesc rdesc = rop.getConf();
- int tag = rdesc.getTag();
-
- int index = -1;
- for (int i = 0; i < order.length; i++)
- if (order[i] == tag) {
- index = i;
- break;
- }
- lops[index] = parents.get(hiveParents.indexOf(op));
- ops[index] = op;
- }
-
- parents.clear();
- hiveParents.clear();
-
- for (int i = 0; i < lops.length; i++) {
- parents.add(lops[i]);
- hiveParents.add(ops[i]);
- }
- }
-
- /**
- * translate a hive join operator to asterix join operator->assign
- * operator->project operator
- *
- * @param parentOperator
- * @param operator
- * @return
- */
- private ILogicalOperator translateJoinOperator(Operator operator, Mutable<ILogicalOperator> parentOperator,
- Translator t) {
-
- JoinDesc joinDesc = (JoinDesc) operator.getConf();
-
- // get the projection expression (already re-written) from each source
- // table
- Map<Byte, List<ExprNodeDesc>> exprMap = joinDesc.getExprs();
- reorder(joinDesc.getTagOrder(), operatorToAsterixParents.get(operator), operatorToHiveParents.get(operator));
-
- // make an reduce join operator
- ILogicalOperator currentOperator = generateJoinTree(joinDesc.getCondsList(),
- operatorToAsterixParents.get(operator), operatorToHiveParents.get(operator), 0, t);
- parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-
- // add assign and project operator on top of a join
- // output variables
- ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
- Set<Entry<Byte, List<ExprNodeDesc>>> entries = exprMap.entrySet();
- Iterator<Entry<Byte, List<ExprNodeDesc>>> iterator = entries.iterator();
- while (iterator.hasNext()) {
- List<ExprNodeDesc> outputExprs = iterator.next().getValue();
- ILogicalOperator assignOperator = t.getAssignOperator(parentOperator, outputExprs, variables);
-
- if (assignOperator != null) {
- currentOperator = assignOperator;
- parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
- }
- }
-
- ILogicalOperator po = new ProjectOperator(variables);
- po.getInputs().add(parentOperator);
- t.rewriteOperatorOutputSchema(variables, operator);
- return po;
- }
-
- /**
- * deal with reduce sink operator for the case of join
- */
- @Override
- public Mutable<ILogicalOperator> visit(ReduceSinkOperator operator, Mutable<ILogicalOperator> parentOperator,
- Translator t) {
-
- Operator downStream = (Operator) operator.getChildOperators().get(0);
- if (!(downStream instanceof JoinOperator))
- return null;
-
- ReduceSinkDesc desc = (ReduceSinkDesc) operator.getConf();
- List<ExprNodeDesc> keys = desc.getKeyCols();
- List<ExprNodeDesc> values = desc.getValueCols();
- List<ExprNodeDesc> partitionCols = desc.getPartitionCols();
-
- /**
- * rewrite key, value, paritioncol expressions
- */
- for (ExprNodeDesc key : keys)
- t.rewriteExpression(key);
- for (ExprNodeDesc value : values)
- t.rewriteExpression(value);
- for (ExprNodeDesc col : partitionCols)
- t.rewriteExpression(col);
-
- ILogicalOperator currentOperator = null;
-
- // add assign operator for keys if necessary
- ArrayList<LogicalVariable> keyVariables = new ArrayList<LogicalVariable>();
- ILogicalOperator assignOperator = t.getAssignOperator(parentOperator, keys, keyVariables);
- if (assignOperator != null) {
- currentOperator = assignOperator;
- parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- // add assign operator for values if necessary
- ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
- assignOperator = t.getAssignOperator(parentOperator, values, variables);
- if (assignOperator != null) {
- currentOperator = assignOperator;
- parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- // unified schema: key, value
- ArrayList<LogicalVariable> unifiedKeyValues = new ArrayList<LogicalVariable>();
- unifiedKeyValues.addAll(keyVariables);
- for (LogicalVariable value : variables)
- if (keyVariables.indexOf(value) < 0)
- unifiedKeyValues.add(value);
-
- // insert projection operator, it is a *must*,
- // in hive, reduce sink sometimes also do the projection operator's
- // task
- currentOperator = new ProjectOperator(unifiedKeyValues);
- currentOperator.getInputs().add(parentOperator);
- parentOperator = new MutableObject<ILogicalOperator>(currentOperator);
-
- reduceSinkToKeyVariables.put(operator, keyVariables);
- List<String> fieldNames = new ArrayList<String>();
- List<TypeInfo> types = new ArrayList<TypeInfo>();
- for (LogicalVariable var : unifiedKeyValues) {
- fieldNames.add(var.toString());
- types.add(t.getType(var));
- }
- reduceSinkToFieldNames.put(operator, fieldNames);
- reduceSinkToTypes.put(operator, types);
- t.rewriteOperatorOutputSchema(variables, operator);
-
- latestAlgebricksOperator = currentOperator;
- latestReduceSink = operator;
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- /**
- * partial rewrite a join operator
- *
- * @param operator
- * @param t
- */
- private void translateJoinOperatorPreprocess(Operator operator, Translator t) {
- JoinDesc desc = (JoinDesc) operator.getConf();
- ReduceSinkDesc reduceSinkDesc = (ReduceSinkDesc) latestReduceSink.getConf();
- int tag = reduceSinkDesc.getTag();
-
- Map<Byte, List<ExprNodeDesc>> exprMap = desc.getExprs();
- List<ExprNodeDesc> exprs = exprMap.get(Byte.valueOf((byte) tag));
-
- for (ExprNodeDesc expr : exprs)
- t.rewriteExpression(expr);
-
- List<Operator> parents = operatorToHiveParents.get(operator);
- if (parents == null) {
- parents = new ArrayList<Operator>();
- operatorToHiveParents.put(operator, parents);
- }
- parents.add(latestReduceSink);
-
- List<ILogicalOperator> asterixParents = operatorToAsterixParents.get(operator);
- if (asterixParents == null) {
- asterixParents = new ArrayList<ILogicalOperator>();
- operatorToAsterixParents.put(operator, asterixParents);
- }
- asterixParents.add(latestAlgebricksOperator);
- }
-
- // generate a join tree from a list of exchange/reducesink operator
- // both exchanges and reduce sinks have the same order
- private ILogicalOperator generateJoinTree(List<JoinCondDesc> conds, List<ILogicalOperator> exchanges,
- List<Operator> reduceSinks, int offset, Translator t) {
- // get a list of reduce sink descs (input descs)
- int inputSize = reduceSinks.size() - offset;
-
- if (inputSize == 2) {
- ILogicalOperator currentRoot;
-
- List<ReduceSinkDesc> reduceSinkDescs = new ArrayList<ReduceSinkDesc>();
- for (int i = reduceSinks.size() - 1; i >= offset; i--)
- reduceSinkDescs.add((ReduceSinkDesc) reduceSinks.get(i).getConf());
-
- // get the object inspector for the join
- List<String> fieldNames = new ArrayList<String>();
- List<TypeInfo> types = new ArrayList<TypeInfo>();
- for (int i = reduceSinks.size() - 1; i >= offset; i--) {
- fieldNames.addAll(reduceSinkToFieldNames.get(reduceSinks.get(i)));
- types.addAll(reduceSinkToTypes.get(reduceSinks.get(i)));
- }
-
- // get number of equality conjunctions in the final join condition
- int size = reduceSinkDescs.get(0).getKeyCols().size();
-
- // make up the join conditon expression
- List<ExprNodeDesc> joinConditionChildren = new ArrayList<ExprNodeDesc>();
- for (int i = 0; i < size; i++) {
- // create a join key pair
- List<ExprNodeDesc> keyPair = new ArrayList<ExprNodeDesc>();
- for (ReduceSinkDesc sink : reduceSinkDescs) {
- keyPair.add(sink.getKeyCols().get(i));
- }
- // create a hive equal condition
- ExprNodeDesc equality = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
- new GenericUDFOPEqual(), keyPair);
- // add the equal condition to the conjunction list
- joinConditionChildren.add(equality);
- }
- // get final conjunction expression
- ExprNodeDesc conjunct = null;
-
- if (joinConditionChildren.size() > 1)
- conjunct = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
- joinConditionChildren);
- else if (joinConditionChildren.size() == 1)
- conjunct = joinConditionChildren.get(0);
- else {
- // there is no join equality condition, equal-join
- conjunct = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, new Boolean(true));
- }
- // get an ILogicalExpression from hive's expression
- Mutable<ILogicalExpression> expression = t.translateScalarFucntion(conjunct);
-
- Mutable<ILogicalOperator> leftBranch = new MutableObject<ILogicalOperator>(
- exchanges.get(exchanges.size() - 1));
- Mutable<ILogicalOperator> rightBranch = new MutableObject<ILogicalOperator>(
- exchanges.get(exchanges.size() - 2));
- // get the join operator
- if (conds.get(offset).getType() == JoinDesc.LEFT_OUTER_JOIN) {
- currentRoot = new LeftOuterJoinOperator(expression);
- Mutable<ILogicalOperator> temp = leftBranch;
- leftBranch = rightBranch;
- rightBranch = temp;
- } else if (conds.get(offset).getType() == JoinDesc.RIGHT_OUTER_JOIN) {
- currentRoot = new LeftOuterJoinOperator(expression);
- } else
- currentRoot = new InnerJoinOperator(expression);
-
- currentRoot.getInputs().add(leftBranch);
- currentRoot.getInputs().add(rightBranch);
-
- // rewriteOperatorOutputSchema(variables, operator);
- return currentRoot;
- } else {
- // get the child join operator and insert and one-to-one exchange
- ILogicalOperator joinSrcOne = generateJoinTree(conds, exchanges, reduceSinks, offset + 1, t);
- // joinSrcOne.addInput(childJoin);
-
- ILogicalOperator currentRoot;
-
- List<ReduceSinkDesc> reduceSinkDescs = new ArrayList<ReduceSinkDesc>();
- for (int i = offset; i < offset + 2; i++)
- reduceSinkDescs.add((ReduceSinkDesc) reduceSinks.get(i).getConf());
-
- // get the object inspector for the join
- List<String> fieldNames = new ArrayList<String>();
- List<TypeInfo> types = new ArrayList<TypeInfo>();
- for (int i = offset; i < reduceSinks.size(); i++) {
- fieldNames.addAll(reduceSinkToFieldNames.get(reduceSinks.get(i)));
- types.addAll(reduceSinkToTypes.get(reduceSinks.get(i)));
- }
-
- // get number of equality conjunctions in the final join condition
- int size = reduceSinkDescs.get(0).getKeyCols().size();
-
- // make up the join condition expression
- List<ExprNodeDesc> joinConditionChildren = new ArrayList<ExprNodeDesc>();
- for (int i = 0; i < size; i++) {
- // create a join key pair
- List<ExprNodeDesc> keyPair = new ArrayList<ExprNodeDesc>();
- for (ReduceSinkDesc sink : reduceSinkDescs) {
- keyPair.add(sink.getKeyCols().get(i));
- }
- // create a hive equal condition
- ExprNodeDesc equality = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
- new GenericUDFOPEqual(), keyPair);
- // add the equal condition to the conjunction list
- joinConditionChildren.add(equality);
- }
- // get final conjunction expression
- ExprNodeDesc conjunct = null;
-
- if (joinConditionChildren.size() > 1)
- conjunct = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
- joinConditionChildren);
- else if (joinConditionChildren.size() == 1)
- conjunct = joinConditionChildren.get(0);
- else {
- // there is no join equality condition, full outer join
- conjunct = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, new Boolean(true));
- }
- // get an ILogicalExpression from hive's expression
- Mutable<ILogicalExpression> expression = t.translateScalarFucntion(conjunct);
-
- Mutable<ILogicalOperator> leftBranch = new MutableObject<ILogicalOperator>(joinSrcOne);
- Mutable<ILogicalOperator> rightBranch = new MutableObject<ILogicalOperator>(exchanges.get(offset));
-
- // get the join operator
- if (conds.get(offset).getType() == JoinDesc.LEFT_OUTER_JOIN) {
- currentRoot = new LeftOuterJoinOperator(expression);
- Mutable<ILogicalOperator> temp = leftBranch;
- leftBranch = rightBranch;
- rightBranch = temp;
- } else if (conds.get(offset).getType() == JoinDesc.RIGHT_OUTER_JOIN) {
- currentRoot = new LeftOuterJoinOperator(expression);
- } else
- currentRoot = new InnerJoinOperator(expression);
-
- // set the inputs from Algebricks join operator
- // add the current table
- currentRoot.getInputs().add(leftBranch);
- currentRoot.getInputs().add(rightBranch);
-
- return currentRoot;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java
deleted file mode 100644
index aa1837c..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LateralViewJoinVisitor.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.plan.LateralViewJoinDesc;
-import org.apache.hadoop.hive.ql.plan.UDTFDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-/**
- * The lateral view join operator is used for FROM src LATERAL VIEW udtf()...
- * This operator was implemented with the following operator DAG in mind.
- * For a query such as
- * SELECT pageid, adid.* FROM example_table LATERAL VIEW explode(adid_list) AS
- * adid
- * The top of the operator DAG will look similar to
- * [Table Scan] | [Lateral View Forward] / \ [Select](*) [Select](adid_list) | |
- * | [UDTF] (explode) \ / [Lateral View Join] | | [Select] (pageid, adid.*) |
- * ....
- * Rows from the table scan operator are first to a lateral view forward
- * operator that just forwards the row and marks the start of a LV. The select
- * operator on the left picks all the columns while the select operator on the
- * right picks only the columns needed by the UDTF.
- * The output of select in the left branch and output of the UDTF in the right
- * branch are then sent to the lateral view join (LVJ). In most cases, the UDTF
- * will generate > 1 row for every row received from the TS, while the left
- * select operator will generate only one. For each row output from the TS, the
- * LVJ outputs all possible rows that can be created by joining the row from the
- * left select and one of the rows output from the UDTF.
- * Additional lateral views can be supported by adding a similar DAG after the
- * previous LVJ operator.
- */
-
-@SuppressWarnings("rawtypes")
-public class LateralViewJoinVisitor extends DefaultVisitor {
-
- private UDTFDesc udtf;
-
- private List<Mutable<ILogicalOperator>> parents = new ArrayList<Mutable<ILogicalOperator>>();
-
- @Override
- public Mutable<ILogicalOperator> visit(LateralViewJoinOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException {
-
- parents.add(AlgebricksParentOperatorRef);
- if (operator.getParentOperators().size() > parents.size()) {
- return null;
- }
-
- ILogicalOperator parentOperator = null;
- ILogicalExpression unnestArg = null;
- List<LogicalVariable> projectVariables = new ArrayList<LogicalVariable>();
- for (Mutable<ILogicalOperator> parentLOpRef : parents) {
- VariableUtilities.getLiveVariables(parentLOpRef.getValue(), projectVariables);
- }
- for (Operator parentOp : operator.getParentOperators()) {
- if (parentOp instanceof UDTFOperator) {
- int index = operator.getParentOperators().indexOf(parentOp);
- List<LogicalVariable> unnestVars = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(parents.get(index).getValue(), unnestVars);
- unnestArg = new VariableReferenceExpression(unnestVars.get(0));
- parentOperator = parents.get(index).getValue();
- }
- }
-
- LogicalVariable var = t.getVariable(udtf.toString(), TypeInfoFactory.unknownTypeInfo);
- Mutable<ILogicalExpression> unnestExpr = t.translateUnnestFunction(udtf, new MutableObject<ILogicalExpression>(
- unnestArg));
- ILogicalOperator currentOperator = new UnnestOperator(var, unnestExpr);
-
- List<LogicalVariable> outputVars = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(parents.get(0).getValue(), outputVars);
- outputVars.add(var);
- ILogicalOperator inputProjectOperator = new ProjectOperator(projectVariables);
- currentOperator.getInputs().add(new MutableObject<ILogicalOperator>(inputProjectOperator));
- inputProjectOperator.getInputs().addAll(parentOperator.getInputs());
-
- parents.clear();
- udtf = null;
- List<ColumnInfo> inputSchema = operator.getSchema().getSignature();
- rewriteOperatorDesc(outputVars, operator.getConf(), inputSchema, t);
- //t.rewriteOperatorOutputSchema(outputVars, operator);
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(UDTFOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) {
- Schema currentSchema = t.generateInputSchema(operator.getParentOperators().get(0));
- udtf = (UDTFDesc) operator.getConf();
-
- // populate the schema from upstream operator
- operator.setSchema(operator.getParentOperators().get(0).getSchema());
- List<LogicalVariable> latestOutputSchema = t.getVariablesFromSchema(currentSchema);
- t.rewriteOperatorOutputSchema(latestOutputSchema, operator);
- return null;
- }
-
- private void rewriteOperatorDesc(List<LogicalVariable> variables, LateralViewJoinDesc desc,
- List<ColumnInfo> schema, Translator t) {
- List<String> outputFieldNames = desc.getOutputInternalColNames();
- for (int i = 0; i < variables.size(); i++) {
- LogicalVariable var = variables.get(i);
- String fieldName = outputFieldNames.get(i);
- String tabAlias = findTabAlias(fieldName, schema);
- fieldName = tabAlias + "." + fieldName;
- if (fieldName.indexOf("$$") < 0) {
- //outputFieldNames.set(i, var.toString());
- t.updateVariable(fieldName, var);
- }
- }
- }
-
- private String findTabAlias(String fieldName, List<ColumnInfo> schema) {
- for (int i = 0; i < schema.size(); i++) {
- ColumnInfo column = schema.get(i);
- if (column.getInternalName().equals(fieldName)) {
- return column.getTabAlias();
- }
- }
- return "null";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java
deleted file mode 100644
index 0876642..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/LimitVisitor.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.plan.LimitDesc;
-
-import edu.uci.ics.hivesterix.logical.expression.HivesterixConstantValue;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-
-public class LimitVisitor extends DefaultVisitor {
-
- @Override
- public Mutable<ILogicalOperator> visit(LimitOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) {
- Schema currentSchema = t.generateInputSchema(operator.getParentOperators().get(0));
-
- LimitDesc desc = (LimitDesc) operator.getConf();
- int limit = desc.getLimit();
- Integer limitValue = new Integer(limit);
-
- ILogicalExpression expr = new ConstantExpression(new HivesterixConstantValue(limitValue));
- ILogicalOperator currentOperator = new edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator(
- expr, true);
- currentOperator.getInputs().add(AlgebricksParentOperatorRef);
-
- operator.setSchema(operator.getParentOperators().get(0).getSchema());
- List<LogicalVariable> latestOutputSchema = t.getVariablesFromSchema(currentSchema);
- t.rewriteOperatorOutputSchema(latestOutputSchema, operator);
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java
deleted file mode 100644
index 3ed9786..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/MapJoinVisitor.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-
-@SuppressWarnings("rawtypes")
-public class MapJoinVisitor extends DefaultVisitor {
-
- /**
- * map a join operator (in hive) to its parent operators (in asterix)
- */
- private HashMap<Operator, List<Mutable<ILogicalOperator>>> opMap = new HashMap<Operator, List<Mutable<ILogicalOperator>>>();
-
- @Override
- public Mutable<ILogicalOperator> visit(MapJoinOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) {
- List<Operator<? extends OperatorDesc>> joinSrc = operator.getParentOperators();
- List<Mutable<ILogicalOperator>> parents = opMap.get(operator);
- if (parents == null) {
- parents = new ArrayList<Mutable<ILogicalOperator>>();
- opMap.put(operator, parents);
- }
- parents.add(AlgebricksParentOperatorRef);
- if (joinSrc.size() != parents.size())
- return null;
-
- ILogicalOperator currentOperator;
- // make an map join operator
- // TODO: will have trouble for n-way joins
- MapJoinDesc joinDesc = (MapJoinDesc) operator.getConf();
-
- Map<Byte, List<ExprNodeDesc>> keyMap = joinDesc.getKeys();
- // get the projection expression (already re-written) from each source
- // table
- Map<Byte, List<ExprNodeDesc>> exprMap = joinDesc.getExprs();
-
- int inputSize = operator.getParentOperators().size();
- // get a list of reduce sink descs (input descs)
-
- // get the parent operator
- List<Mutable<ILogicalOperator>> parentOps = parents;
-
- List<String> fieldNames = new ArrayList<String>();
- List<TypeInfo> types = new ArrayList<TypeInfo>();
- for (Operator ts : joinSrc) {
- List<ColumnInfo> columns = ts.getSchema().getSignature();
- for (ColumnInfo col : columns) {
- fieldNames.add(col.getInternalName());
- types.add(col.getType());
- }
- }
-
- // get number of equality conjunctions in the final join condition
- Set<Entry<Byte, List<ExprNodeDesc>>> keyEntries = keyMap.entrySet();
- Iterator<Entry<Byte, List<ExprNodeDesc>>> entry = keyEntries.iterator();
-
- int size = 0;
- if (entry.hasNext())
- size = entry.next().getValue().size();
-
- // make up the join conditon expression
- List<ExprNodeDesc> joinConditionChildren = new ArrayList<ExprNodeDesc>();
- for (int i = 0; i < size; i++) {
- // create a join key pair
- List<ExprNodeDesc> keyPair = new ArrayList<ExprNodeDesc>();
- for (int j = 0; j < inputSize; j++) {
- keyPair.add(keyMap.get(Byte.valueOf((byte) j)).get(i));
- }
- // create a hive equal condition
- ExprNodeDesc equality = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo,
- new GenericUDFOPEqual(), keyPair);
- // add the equal condition to the conjunction list
- joinConditionChildren.add(equality);
- }
- // get final conjunction expression
- ExprNodeDesc conjunct = null;
-
- if (joinConditionChildren.size() > 1)
- conjunct = new ExprNodeGenericFuncDesc(TypeInfoFactory.booleanTypeInfo, new GenericUDFOPAnd(),
- joinConditionChildren);
- else if (joinConditionChildren.size() == 1)
- conjunct = joinConditionChildren.get(0);
- else {
- // there is no join equality condition, full outer join
- conjunct = new ExprNodeConstantDesc(TypeInfoFactory.booleanTypeInfo, new Boolean(true));
- }
- // get an ILogicalExpression from hive's expression
- Mutable<ILogicalExpression> expression = t.translateScalarFucntion(conjunct);
-
- ArrayList<LogicalVariable> left = new ArrayList<LogicalVariable>();
- ArrayList<LogicalVariable> right = new ArrayList<LogicalVariable>();
-
- Set<Entry<Byte, List<ExprNodeDesc>>> kentries = keyMap.entrySet();
- Iterator<Entry<Byte, List<ExprNodeDesc>>> kiterator = kentries.iterator();
- int iteration = 0;
- ILogicalOperator assignOperator = null;
- while (kiterator.hasNext()) {
- List<ExprNodeDesc> outputExprs = kiterator.next().getValue();
-
- if (iteration == 0)
- assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, outputExprs, left);
- else
- assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, outputExprs, right);
-
- if (assignOperator != null) {
- currentOperator = assignOperator;
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
- }
- iteration++;
- }
-
- List<Mutable<ILogicalOperator>> inputs = parentOps;
-
- // get the join operator
- currentOperator = new InnerJoinOperator(expression);
-
- // set the inputs from asterix join operator
- for (Mutable<ILogicalOperator> input : inputs)
- currentOperator.getInputs().add(input);
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-
- // add assign and project operator
- // output variables
- ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
- Set<Entry<Byte, List<ExprNodeDesc>>> entries = exprMap.entrySet();
- Iterator<Entry<Byte, List<ExprNodeDesc>>> iterator = entries.iterator();
- while (iterator.hasNext()) {
- List<ExprNodeDesc> outputExprs = iterator.next().getValue();
- assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, outputExprs, variables);
-
- if (assignOperator != null) {
- currentOperator = assignOperator;
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
- }
- }
-
- currentOperator = new ProjectOperator(variables);
- currentOperator.getInputs().add(AlgebricksParentOperatorRef);
- t.rewriteOperatorOutputSchema(variables, operator);
- // opMap.clear();
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java
deleted file mode 100644
index d69fb1c..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/ProjectVisitor.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.SelectDesc;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-
-public class ProjectVisitor extends DefaultVisitor {
-
- /**
- * translate project operator
- */
- @Override
- public Mutable<ILogicalOperator> visit(SelectOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
- Translator t) {
-
- SelectDesc desc = (SelectDesc) operator.getConf();
-
- if (desc == null)
- return null;
-
- List<ExprNodeDesc> cols = desc.getColList();
-
- if (cols == null)
- return null;
-
- // insert assign operator if necessary
- ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-
- for (ExprNodeDesc expr : cols)
- t.rewriteExpression(expr);
-
- ILogicalOperator assignOp = t.getAssignOperator(AlgebricksParentOperator, cols, variables);
- ILogicalOperator currentOperator = null;
- if (assignOp != null) {
- currentOperator = assignOp;
- AlgebricksParentOperator = new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- currentOperator = new ProjectOperator(variables);
- currentOperator.getInputs().add(AlgebricksParentOperator);
- t.rewriteOperatorOutputSchema(variables, operator);
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java
deleted file mode 100644
index 277af50..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/SortVisitor.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
-
-public class SortVisitor extends DefaultVisitor {
-
- @SuppressWarnings("rawtypes")
- @Override
- public Mutable<ILogicalOperator> visit(ReduceSinkOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException {
- ReduceSinkDesc desc = (ReduceSinkDesc) operator.getConf();
- Operator downStream = (Operator) operator.getChildOperators().get(0);
- List<ExprNodeDesc> keys = desc.getKeyCols();
- if (!(downStream instanceof ExtractOperator && desc.getNumReducers() == 1 && keys.size() > 0)) {
- return null;
- }
-
- List<ExprNodeDesc> schema = new ArrayList<ExprNodeDesc>();
- List<ExprNodeDesc> values = desc.getValueCols();
- List<ExprNodeDesc> partitionCols = desc.getPartitionCols();
- for (ExprNodeDesc key : keys) {
- t.rewriteExpression(key);
- }
- for (ExprNodeDesc value : values) {
- t.rewriteExpression(value);
- }
- for (ExprNodeDesc col : partitionCols) {
- t.rewriteExpression(col);
- }
-
- // add a order-by operator and limit if any
- List<Pair<IOrder, Mutable<ILogicalExpression>>> pairs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
- char[] orders = desc.getOrder().toCharArray();
- int i = 0;
- for (ExprNodeDesc key : keys) {
- Mutable<ILogicalExpression> expr = t.translateScalarFucntion(key);
- IOrder order = orders[i] == '+' ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
-
- Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<IOrder, Mutable<ILogicalExpression>>(order, expr);
- pairs.add(pair);
- i++;
- }
-
- // get input variables
- ArrayList<LogicalVariable> inputVariables = new ArrayList<LogicalVariable>();
- VariableUtilities.getProducedVariables(AlgebricksParentOperatorRef.getValue(), inputVariables);
-
- ArrayList<LogicalVariable> keyVariables = new ArrayList<LogicalVariable>();
- ILogicalOperator currentOperator;
- ILogicalOperator assignOp = t.getAssignOperator(AlgebricksParentOperatorRef, keys, keyVariables);
- if (assignOp != null) {
- currentOperator = assignOp;
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- OrderColumn[] keyColumns = new OrderColumn[keyVariables.size()];
-
- for (int j = 0; j < keyColumns.length; j++)
- keyColumns[j] = new OrderColumn(keyVariables.get(j), pairs.get(j).first.getKind());
-
- // handle order operator
- currentOperator = new OrderOperator(pairs);
- currentOperator.getInputs().add(AlgebricksParentOperatorRef);
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
-
- // project back, remove generated sort-key columns if any
- if (assignOp != null) {
- currentOperator = new ProjectOperator(inputVariables);
- currentOperator.getInputs().add(AlgebricksParentOperatorRef);
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- /**
- * a special rule for hive's order by output schema of reduce sink
- * operator only contains the columns
- */
- for (ExprNodeDesc value : values) {
- schema.add(value);
- }
-
- ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
- ILogicalOperator assignOperator = t.getAssignOperator(AlgebricksParentOperatorRef, schema, variables);
- t.rewriteOperatorOutputSchema(variables, operator);
-
- if (assignOperator != null) {
- currentOperator = assignOperator;
- AlgebricksParentOperatorRef = new MutableObject<ILogicalOperator>(currentOperator);
- }
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java
deleted file mode 100644
index 25abdec..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/TableScanWriteVisitor.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveDataSink;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveDataSource;
-import edu.uci.ics.hivesterix.runtime.jobgen.HiveMetaDataProvider;
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-public class TableScanWriteVisitor extends DefaultVisitor {
-
- /**
- * map from alias to partition desc
- */
- private HashMap<String, PartitionDesc> aliasToPathMap;
-
- /**
- * map from partition desc to data source
- */
- private HashMap<PartitionDesc, IDataSource<PartitionDesc>> dataSourceMap = new HashMap<PartitionDesc, IDataSource<PartitionDesc>>();
-
- /**
- * constructor
- *
- * @param aliasToPathMap
- */
- public TableScanWriteVisitor(HashMap<String, PartitionDesc> aliasToPathMap) {
- this.aliasToPathMap = aliasToPathMap;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(TableScanOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- TableScanDesc desc = (TableScanDesc) operator.getConf();
- if (desc == null || desc.getAlias()==null) {
- List<LogicalVariable> schema = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(AlgebricksParentOperator.getValue(), schema);
- t.rewriteOperatorOutputSchema(schema, operator);
- return null;
- }
-
- List<ColumnInfo> columns = operator.getSchema().getSignature();
- for (int i = columns.size() - 1; i >= 0; i--)
- if (columns.get(i).getIsVirtualCol() == true)
- columns.remove(i);
-
- // start with empty tuple operator
- List<TypeInfo> types = new ArrayList<TypeInfo>();
- ArrayList<LogicalVariable> variables = new ArrayList<LogicalVariable>();
- List<String> names = new ArrayList<String>();
- for (ColumnInfo column : columns) {
- types.add(column.getType());
-
- LogicalVariable var = t.getVariableFromFieldName(column.getTabAlias() + "." + column.getInternalName());
- LogicalVariable varNew;
-
- if (var != null) {
- varNew = t.getVariable(column.getTabAlias() + "." + column.getInternalName() + operator.toString(),
- column.getType());
- t.replaceVariable(var, varNew);
- var = varNew;
- } else
- var = t.getNewVariable(column.getTabAlias() + "." + column.getInternalName(), column.getType());
-
- variables.add(var);
- names.add(column.getInternalName());
- }
- Schema currentSchema = new Schema(names, types);
-
- String alias = desc.getAlias();
- PartitionDesc partDesc = aliasToPathMap.get(alias);
- IDataSource<PartitionDesc> dataSource = new HiveDataSource<PartitionDesc>(partDesc, currentSchema.getSchema());
- ILogicalOperator currentOperator = new DataSourceScanOperator(variables, dataSource);
-
- // set empty tuple source operator
- ILogicalOperator ets = new EmptyTupleSourceOperator();
- currentOperator.getInputs().add(new MutableObject<ILogicalOperator>(ets));
-
- // setup data source
- dataSourceMap.put(partDesc, dataSource);
- t.rewriteOperatorOutputSchema(variables, operator);
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(FileSinkOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) {
- if (hiveOperator.getChildOperators() != null && hiveOperator.getChildOperators().size() > 0)
- return null;
-
- Schema currentSchema = t.generateInputSchema(hiveOperator.getParentOperators().get(0));
-
- IDataSink sink = new HiveDataSink(hiveOperator, currentSchema.getSchema());
- List<Mutable<ILogicalExpression>> exprList = new ArrayList<Mutable<ILogicalExpression>>();
- for (String column : currentSchema.getNames()) {
- exprList.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(t.getVariable(column))));
- }
-
- ILogicalOperator currentOperator = new WriteOperator(exprList, sink);
- if (AlgebricksParentOperator != null) {
- currentOperator.getInputs().add(AlgebricksParentOperator);
- }
-
- IMetadataProvider<PartitionDesc, Object> metaData = new HiveMetaDataProvider<PartitionDesc, Object>(
- hiveOperator, currentSchema, dataSourceMap);
- t.setMetadataProvider(metaData);
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java
deleted file mode 100644
index c710f3f..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/UnionVisitor.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.DefaultVisitor;
-import edu.uci.ics.hivesterix.logical.plan.visitor.base.Translator;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-
-public class UnionVisitor extends DefaultVisitor {
-
- List<Mutable<ILogicalOperator>> parents = new ArrayList<Mutable<ILogicalOperator>>();
-
- @Override
- public Mutable<ILogicalOperator> visit(UnionOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
- Translator t) throws AlgebricksException {
-
- parents.add(AlgebricksParentOperator);
- if (operator.getParentOperators().size() > parents.size()) {
- return null;
- }
-
- List<LogicalVariable> leftVars = new ArrayList<LogicalVariable>();
- List<LogicalVariable> rightVars = new ArrayList<LogicalVariable>();
-
- VariableUtilities.getLiveVariables(parents.get(0).getValue(), leftVars);
- VariableUtilities.getLiveVariables(parents.get(1).getValue(), rightVars);
-
- List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> triples = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>();
- List<LogicalVariable> unionVars = new ArrayList<LogicalVariable>();
-
- for (int i = 0; i < leftVars.size(); i++) {
- LogicalVariable unionVar = t.getVariable(
- leftVars.get(i).getId() + "union" + AlgebricksParentOperator.hashCode(),
- TypeInfoFactory.unknownTypeInfo);
- unionVars.add(unionVar);
- Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
- leftVars.get(i), rightVars.get(i), unionVar);
- t.replaceVariable(leftVars.get(i), unionVar);
- t.replaceVariable(rightVars.get(i), unionVar);
- triples.add(triple);
- }
- ILogicalOperator currentOperator = new edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator(
- triples);
- for (Mutable<ILogicalOperator> parent : parents)
- currentOperator.getInputs().add(parent);
-
- t.rewriteOperatorOutputSchema(unionVars, operator);
- parents.clear();
- return new MutableObject<ILogicalOperator>(currentOperator);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java
deleted file mode 100644
index f79b79f..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/DefaultVisitor.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor.base;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hadoop.hive.ql.exec.CollectOperator;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.ForwardOperator;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MapOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.ScriptOperator;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-
-/**
- * a default empty implementation of visitor
- *
- * @author yingyib
- */
-public class DefaultVisitor implements Visitor {
-
- @Override
- public Mutable<ILogicalOperator> visit(CollectOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(JoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(ExtractOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(MapJoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(SMBMapJoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- public Mutable<ILogicalOperator> visit(FileSinkOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- public Mutable<ILogicalOperator> visit(ReduceSinkOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(FilterOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(ForwardOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(GroupByOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(LateralViewForwardOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(LateralViewJoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(LimitOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(MapOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(ScriptOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(SelectOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(TableScanOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperator, Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(UDTFOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
- Translator t) throws AlgebricksException {
- return null;
- }
-
- @Override
- public Mutable<ILogicalOperator> visit(UnionOperator operator, Mutable<ILogicalOperator> AlgebricksParentOperator,
- Translator t) throws AlgebricksException {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java
deleted file mode 100644
index 5b6ac50..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Translator.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor.base;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.plan.AggregationDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.PartitionDesc;
-import org.apache.hadoop.hive.ql.plan.UDTFDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-
-import edu.uci.ics.hivesterix.runtime.jobgen.Schema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
-
-@SuppressWarnings("rawtypes")
-public interface Translator {
-
- /**
- * generate input schema
- *
- * @param operator
- * @return
- */
- public Schema generateInputSchema(Operator operator);
-
- /**
- * rewrite the names of output columns for feture expression evaluators to
- * use
- *
- * @param operator
- */
- public void rewriteOperatorOutputSchema(List<LogicalVariable> vars, Operator operator);
-
- /**
- * rewrite the names of output columns for feture expression evaluators to
- * use
- *
- * @param operator
- */
- public void rewriteOperatorOutputSchema(Operator operator);
-
- /**
- * rewrite an expression and substitute variables
- *
- * @param expr
- * hive expression
- */
- public void rewriteExpression(ExprNodeDesc expr);
-
- /**
- * rewrite an expression and substitute variables
- *
- * @param expr
- * hive expression
- */
- public void rewriteExpressionPartial(ExprNodeDesc expr);
-
- /**
- * get an assign operator as a child of parent
- *
- * @param parent
- * @param cols
- * @param variables
- * @return
- */
- public ILogicalOperator getAssignOperator(Mutable<ILogicalOperator> parent, List<ExprNodeDesc> cols,
- ArrayList<LogicalVariable> variables);
-
- /**
- * get type for a logical variable
- *
- * @param var
- * @return type info
- */
- public TypeInfo getType(LogicalVariable var);
-
- /**
- * translate an expression from hive to Algebricks
- *
- * @param desc
- * @return
- */
- public Mutable<ILogicalExpression> translateScalarFucntion(ExprNodeDesc desc);
-
- /**
- * translate an aggregation from hive to Algebricks
- *
- * @param aggregateDesc
- * @return
- */
- public Mutable<ILogicalExpression> translateAggregation(AggregationDesc aggregateDesc);
-
- /**
- * translate unnesting (UDTF) function expression
- *
- * @param aggregator
- * @return
- */
- public Mutable<ILogicalExpression> translateUnnestFunction(UDTFDesc udtfDesc, Mutable<ILogicalExpression> argument);
-
- /**
- * get variable from a schema
- *
- * @param schema
- * @return
- */
- public List<LogicalVariable> getVariablesFromSchema(Schema schema);
-
- /**
- * get variable from name
- *
- * @param name
- * @return
- */
- public LogicalVariable getVariable(String name);
-
- /**
- * get variable from field name
- *
- * @param name
- * @return
- */
- public LogicalVariable getVariableFromFieldName(String name);
-
- /**
- * get variable from name, type
- *
- * @param fieldName
- * @param type
- * @return
- */
- public LogicalVariable getVariable(String fieldName, TypeInfo type);
-
- /**
- * get new variable from name, type
- *
- * @param fieldName
- * @param type
- * @return
- */
- public LogicalVariable getNewVariable(String fieldName, TypeInfo type);
-
- /**
- * update a name-variable binding
- *
- * @param fieldName
- * @param variable
- */
- public void updateVariable(String fieldName, LogicalVariable variable);
-
- /**
- * set the metadata provider
- *
- * @param metadata
- */
- public void setMetadataProvider(IMetadataProvider<PartitionDesc, Object> metadata);
-
- /**
- * get the metadata provider
- *
- * @param metadata
- */
- public IMetadataProvider<PartitionDesc, Object> getMetadataProvider();
-
- /**
- * replace the variable
- *
- * @param oldVar
- * @param newVar
- */
- public void replaceVariable(LogicalVariable oldVar, LogicalVariable newVar);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java
----------------------------------------------------------------------
diff --git a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java b/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java
deleted file mode 100644
index 2291d5f..0000000
--- a/hivesterix/hivesterix-translator/src/main/java/edu/uci/ics/hivesterix/logical/plan/visitor/base/Visitor.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hivesterix.logical.plan.visitor.base;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hadoop.hive.ql.exec.CollectOperator;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.ForwardOperator;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
-import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
-import org.apache.hadoop.hive.ql.exec.LimitOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.MapOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.ScriptOperator;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.UDTFOperator;
-import org.apache.hadoop.hive.ql.exec.UnionOperator;
-
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-
-public interface Visitor {
-
- public Mutable<ILogicalOperator> visit(CollectOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(JoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(ExtractOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(MapJoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(SMBMapJoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(FilterOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(ForwardOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(GroupByOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(LateralViewForwardOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(LateralViewJoinOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(LimitOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(MapOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(ScriptOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(SelectOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(TableScanOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(FileSinkOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(ReduceSinkOperator hiveOperator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(UDTFOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-
- public Mutable<ILogicalOperator> visit(UnionOperator operator,
- Mutable<ILogicalOperator> AlgebricksParentOperatorRef, Translator t) throws AlgebricksException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/ffc967fd/hivesterix/pom.xml
----------------------------------------------------------------------
diff --git a/hivesterix/pom.xml b/hivesterix/pom.xml
deleted file mode 100644
index ccaed66..0000000
--- a/hivesterix/pom.xml
+++ /dev/null
@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ! Copyright 2009-2013 by The Regents of the University of California
- ! Licensed under the Apache License, Version 2.0 (the "License");
- ! you may not use this file except in compliance with the License.
- ! you may obtain a copy of the License from
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing, software
- ! distributed under the License is distributed on an "AS IS" BASIS,
- ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ! See the License for the specific language governing permissions and
- ! limitations under the License.
- !-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hivesterix</artifactId>
- <version>0.2.14-SNAPSHOT</version>
- <packaging>pom</packaging>
- <name>hivesterix</name>
-
- <parent>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>fullstack</artifactId>
- <version>0.2.14-SNAPSHOT</version>
- </parent>
-
- <properties>
- <jvm.extraargs />
- </properties>
-
- <profiles>
- <profile>
- <id>macosx</id>
- <activation>
- <os>
- <name>mac os x</name>
- </os>
- <jdk>1.7</jdk>
- </activation>
- <properties>
- <jvm.extraargs>-Djava.nio.channels.spi.SelectorProvider=sun.nio.ch.KQueueSelectorProvider</jvm.extraargs>
- </properties>
- </profile>
- </profiles>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-release-plugin</artifactId>
- <version>2.0</version>
- <configuration>
- <goals>package source:jar javadoc:jar deploy:deploy</goals>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>versions-maven-plugin</artifactId>
- <version>1.2</version>
- </plugin>
- </plugins>
- </build>
-
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-changelog-plugin</artifactId>
- <version>2.2</version>
- </plugin>
- </plugins>
- </reporting>
-
- <distributionManagement>
- <repository>
- <id>hyracks-releases</id>
- <url>http://obelix.ics.uci.edu/nexus/content/repositories/hyracks-releases/</url>
- </repository>
- <snapshotRepository>
- <id>hyracks-snapshots</id>
- <url>http://obelix.ics.uci.edu/nexus/content/repositories/hyracks-snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
-
- <repositories>
- <repository>
- <id>maven-central</id>
- <url>http://repo1.maven.org/maven2/</url>
- </repository>
- <repository>
- <id>hyracks-public</id>
- <url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public/</url>
- </repository>
- <repository>
- <id>jboss-public</id>
- <url>https://repository.jboss.org/nexus/content/groups/public/</url>
- </repository>
- <repository>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- <checksumPolicy>warn</checksumPolicy>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- <checksumPolicy>fail</checksumPolicy>
- </snapshots>
- <id>third-party</id>
- <url>http://obelix.ics.uci.edu/nexus/content/repositories/third-party</url>
- </repository>
- <repository>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- <checksumPolicy>warn</checksumPolicy>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- <checksumPolicy>fail</checksumPolicy>
- </snapshots>
- <id>hyracks-public-release</id>
- <url>http://obelix.ics.uci.edu/nexus/content/repositories/hyracks-public-releases</url>
- </repository>
- </repositories>
-
- <pluginRepositories>
- <pluginRepository>
- <id>hyracks-public</id>
- <url>http://obelix.ics.uci.edu/nexus/content/groups/hyracks-public/</url>
- <releases>
- <updatePolicy>always</updatePolicy>
- </releases>
- </pluginRepository>
- </pluginRepositories>
-
- <modules>
- <module>hivesterix-runtime</module>
- <module>hivesterix-translator</module>
- <module>hivesterix-optimizer</module>
- <module>hivesterix-serde</module>
- <module>hivesterix-dist</module>
- <module>hivesterix-common</module>
- </modules>
-</project>