You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/03/10 05:20:48 UTC
svn commit: r1665403 [4/7] - in /hive/branches/spark: ./
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/ an...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java Tue Mar 10 04:20:45 2015
@@ -416,8 +416,7 @@ public class SortedDynPartitionOptimizer
// should honor the ordering of records provided by ORDER BY in SELECT statement
ReduceSinkOperator parentRSOp = OperatorUtils.findSingleOperatorUpstream(parent,
ReduceSinkOperator.class);
- boolean isOrderBy = parseCtx.getQB().getParseInfo().getDestToOrderBy().size() > 0;
- if (parentRSOp != null && isOrderBy) {
+ if (parentRSOp != null && parseCtx.getQueryProperties().hasOuterOrderBy()) {
String parentRSOpOrder = parentRSOp.getConf().getOrder();
if (parentRSOpOrder != null && !parentRSOpOrder.isEmpty() && sortPositions.isEmpty()) {
newKeyCols.addAll(parentRSOp.getConf().getKeyCols());
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java Tue Mar 10 04:20:45 2015
@@ -97,8 +97,8 @@ public class StatsOptimizer implements T
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
- if (pctx.getFetchTask() != null || !pctx.getQB().getIsQuery() ||
- pctx.getQB().isAnalyzeRewrite() || pctx.getQB().isCTAS() ||
+ if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() ||
+ pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS() ||
pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty()) {
return pctx;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java Tue Mar 10 04:20:45 2015
@@ -213,7 +213,7 @@ class ASTBuilder {
case TIMESTAMP: {
val = literal.getValue();
type = HiveParser.TOK_TIMESTAMPLITERAL;
- DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
val = df.format(((Calendar) val).getTime());
val = "'" + val + "'";
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java Tue Mar 10 04:20:45 2015
@@ -54,10 +54,12 @@ import org.apache.hadoop.hive.ql.parse.S
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
@@ -68,7 +70,10 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -152,8 +157,13 @@ public class RexNodeConverter {
// TODO: 1) Expand to other functions as needed 2) What about types other than primitive.
TypeInfo tgtDT = null;
GenericUDF tgtUdf = func.getGenericUDF();
- boolean isNumeric = tgtUdf instanceof GenericUDFBaseNumeric,
- isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare;
+
+ boolean isNumeric = (tgtUdf instanceof GenericUDFBaseBinary
+ && func.getTypeInfo().getCategory() == Category.PRIMITIVE
+ && (PrimitiveGrouping.NUMERIC_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping(
+ ((PrimitiveTypeInfo) func.getTypeInfo()).getPrimitiveCategory())));
+ boolean isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare;
+
if (isNumeric) {
tgtDT = func.getTypeInfo();
@@ -175,8 +185,7 @@ public class RexNodeConverter {
} else if (isNumeric) {
// For numeric, we'll do minimum necessary cast - if we cast to the type
// of expression, bad things will happen.
- GenericUDFBaseNumeric numericUdf = (GenericUDFBaseNumeric)tgtUdf;
- PrimitiveTypeInfo minArgType = numericUdf.deriveMinArgumentCast(childExpr, tgtDT);
+ PrimitiveTypeInfo minArgType = ExprNodeDescUtils.deriveMinArgumentCast(childExpr, tgtDT);
tmpExprNode = ParseUtils.createConversionCast(childExpr, minArgType);
} else {
throw new AssertionError("Unexpected " + tgtDT + " - not a numeric op or compare");
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -698,10 +698,10 @@ public abstract class BaseSemanticAnalyz
}
/**
- * tableSpec.
+ * TableSpec.
*
*/
- public static class tableSpec {
+ public static class TableSpec {
public String tableName;
public Table tableHandle;
public Map<String, String> partSpec; // has to use LinkedHashMap to enforce order
@@ -711,12 +711,12 @@ public abstract class BaseSemanticAnalyz
public static enum SpecType {TABLE_ONLY, STATIC_PARTITION, DYNAMIC_PARTITION};
public SpecType specType;
- public tableSpec(Hive db, HiveConf conf, ASTNode ast)
+ public TableSpec(Hive db, HiveConf conf, ASTNode ast)
throws SemanticException {
this(db, conf, ast, true, false);
}
- public tableSpec(Hive db, HiveConf conf, String tableName, Map<String, String> partSpec)
+ public TableSpec(Hive db, HiveConf conf, String tableName, Map<String, String> partSpec)
throws HiveException {
this.tableName = tableName;
this.partSpec = partSpec;
@@ -730,7 +730,7 @@ public abstract class BaseSemanticAnalyz
}
}
- public tableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartitionsSpec,
+ public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartitionsSpec,
boolean allowPartialPartitionsSpec) throws SemanticException {
assert (ast.getToken().getType() == HiveParser.TOK_TAB
|| ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION
@@ -869,6 +869,47 @@ public abstract class BaseSemanticAnalyz
}
}
+ public class AnalyzeRewriteContext {
+
+ private String tableName;
+ private List<String> colName;
+ private List<String> colType;
+ private boolean tblLvl;
+
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public List<String> getColName() {
+ return colName;
+ }
+
+ public void setColName(List<String> colName) {
+ this.colName = colName;
+ }
+
+ public boolean isTblLvl() {
+ return tblLvl;
+ }
+
+ public void setTblLvl(boolean isTblLvl) {
+ this.tblLvl = isTblLvl;
+ }
+
+ public List<String> getColType() {
+ return colType;
+ }
+
+ public void setColType(List<String> colType) {
+ this.colType = colType;
+ }
+ }
+
/**
* Gets the lineage information.
*
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Tue Mar 10 04:20:45 2015
@@ -718,7 +718,7 @@ public class CalcitePlanner extends Sema
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE);
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE);
hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
- hepPgmBldr.addRuleInstance(ProjectRemoveRule.NAME_CALC_INSTANCE);
+ hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE);
hepPgm = hepPgmBldr.build();
@@ -792,7 +792,7 @@ public class CalcitePlanner extends Sema
RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
- HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY, true);
+ HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
basePlan = fieldTrimmer.trim(basePlan);
// 5. Rerun PPD through Project as column pruning would have introduced DT
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -427,10 +427,12 @@ public class ColumnStatsSemanticAnalyzer
qb = getQB();
qb.setAnalyzeRewrite(true);
qbp = qb.getParseInfo();
- qbp.setTableName(tbl.getTableName());
- qbp.setTblLvl(isTableLevel);
- qbp.setColName(colNames);
- qbp.setColType(colType);
+ analyzeRewrite = new AnalyzeRewriteContext();
+ analyzeRewrite.setTableName(tbl.getTableName());
+ analyzeRewrite.setTblLvl(isTableLevel);
+ analyzeRewrite.setColName(colNames);
+ analyzeRewrite.setColType(colType);
+ qbp.setAnalyzeRewrite(analyzeRewrite);
initCtx(ctx);
LOG.info("Invoking analyze on rewritten query");
analyzeInternal(rewrittenTree);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -45,6 +45,7 @@ import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -997,7 +998,7 @@ public class DDLSemanticAnalyzer extends
StatsWork statDesc;
if (oldTblPartLoc.equals(newTblPartLoc)) {
// If we're merging to the same location, we can avoid some metastore calls
- tableSpec tablepart = new tableSpec(this.db, conf, root);
+ TableSpec tablepart = new TableSpec(this.db, conf, root);
statDesc = new StatsWork(tablepart);
} else {
statDesc = new StatsWork(ltd);
@@ -1036,7 +1037,7 @@ public class DDLSemanticAnalyzer extends
typeName = indexType.getHandlerClsName();
} else {
try {
- Class.forName(typeName);
+ JavaUtils.loadClass(typeName);
} catch (Exception e) {
throw new SemanticException("class name provided for index handler not found.", e);
}
@@ -1618,7 +1619,7 @@ public class DDLSemanticAnalyzer extends
StatsWork statDesc;
if (oldTblPartLoc.equals(newTblPartLoc)) {
// If we're merging to the same location, we can avoid some metastore calls
- tableSpec tablepart = new tableSpec(db, conf, tableName, partSpec);
+ TableSpec tablepart = new TableSpec(db, conf, tableName, partSpec);
statDesc = new StatsWork(tablepart);
} else {
statDesc = new StatsWork(ltd);
@@ -2626,6 +2627,7 @@ public class DDLSemanticAnalyzer extends
// popular case but that's kinda hacky. Let's not do it for now.
boolean canGroupExprs = ifExists;
+ boolean mustPurge = (ast.getFirstChildWithType(HiveParser.KW_PURGE) != null);
Table tab = getTable(qualified);
Map<Integer, List<ExprNodeGenericFuncDesc>> partSpecs =
getFullPartitionSpecs(ast, tab, canGroupExprs);
@@ -2640,7 +2642,7 @@ public class DDLSemanticAnalyzer extends
addTableDropPartsOutputs(tab, partSpecs.values(), !ifExists, ignoreProtection);
DropTableDesc dropTblDesc =
- new DropTableDesc(getDotName(qualified), partSpecs, expectView, ignoreProtection);
+ new DropTableDesc(getDotName(qualified), partSpecs, expectView, ignoreProtection, mustPurge);
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf));
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -18,13 +18,10 @@
package org.apache.hadoop.hive.ql.parse;
-import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
-
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
import java.net.URI;
-import java.util.List;
import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.fs.FileStatus;
@@ -37,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
import org.apache.hadoop.hive.ql.plan.CopyWork;
/**
@@ -59,7 +57,7 @@ public class ExportSemanticAnalyzer exte
URI toURI = EximUtil.getValidatedURI(conf, tmpPath);
// initialize source table/partition
- tableSpec ts = new tableSpec(db, conf, (ASTNode) tableTree, false, true);
+ TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree, false, true);
EximUtil.validateTable(ts.tableHandle);
try {
FileSystem fs = FileSystem.get(toURI, conf);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Mar 10 04:20:45 2015
@@ -1118,8 +1118,8 @@ partitionLocation
alterStatementSuffixDropPartitions[boolean table]
@init { pushMsg("drop partition statement", state); }
@after { popMsg(state); }
- : KW_DROP ifExists? dropPartitionSpec (COMMA dropPartitionSpec)* ignoreProtection?
- -> { table }? ^(TOK_ALTERTABLE_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection?)
+ : KW_DROP ifExists? dropPartitionSpec (COMMA dropPartitionSpec)* ignoreProtection? KW_PURGE?
+ -> { table }? ^(TOK_ALTERTABLE_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection? KW_PURGE?)
-> ^(TOK_ALTERVIEW_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection?)
;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -199,7 +199,7 @@ public class LoadSemanticAnalyzer extend
}
// initialize destination table/partition
- tableSpec ts = new tableSpec(db, conf, (ASTNode) tableTree);
+ TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree);
if (ts.tableHandle.isOffline()){
throw new SemanticException(
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java Tue Mar 10 04:20:45 2015
@@ -42,6 +42,8 @@ import org.apache.hadoop.hive.ql.hooks.L
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc;
import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
@@ -61,7 +63,7 @@ import org.apache.hadoop.hive.ql.plan.Ta
**/
public class ParseContext {
- private QB qb;
+
private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
private HashMap<TableScanOperator, SampleDesc> opToSamplePruner;
@@ -98,10 +100,14 @@ public class ParseContext {
private FetchTask fetchTask;
private QueryProperties queryProperties;
- private TableDesc fetchTabledesc;
+ private TableDesc fetchTableDesc;
private Operator<?> fetchSource;
private ListSinkOperator fetchSink;
+ private AnalyzeRewriteContext analyzeRewrite;
+ private CreateTableDesc createTableDesc;
+
+
public ParseContext() {
}
@@ -142,7 +148,6 @@ public class ParseContext {
*/
public ParseContext(
HiveConf conf,
- QB qb,
HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner,
HashMap<TableScanOperator, PrunedPartitionList> opToPartList,
HashMap<String, Operator<? extends OperatorDesc>> topOps,
@@ -159,9 +164,9 @@ public class ParseContext {
Map<TableScanOperator, Map<String, ExprNodeDesc>> opToPartToSkewedPruner,
Map<String, ReadEntity> viewAliasToInput,
List<ReduceSinkOperator> reduceSinkOperatorsAddedByEnforceBucketingSorting,
+ AnalyzeRewriteContext analyzeRewrite, CreateTableDesc createTableDesc,
QueryProperties queryProperties) {
this.conf = conf;
- this.qb = qb;
this.opToPartPruner = opToPartPruner;
this.opToPartList = opToPartList;
this.joinOps = joinOps;
@@ -184,25 +189,12 @@ public class ParseContext {
this.viewAliasToInput = viewAliasToInput;
this.reduceSinkOperatorsAddedByEnforceBucketingSorting =
reduceSinkOperatorsAddedByEnforceBucketingSorting;
+ this.analyzeRewrite = analyzeRewrite;
+ this.createTableDesc = createTableDesc;
this.queryProperties = queryProperties;
}
/**
- * @return the qb
- */
- public QB getQB() {
- return qb;
- }
-
- /**
- * @param qb
- * the qb to set
- */
- public void setQB(QB qb) {
- this.qb = qb;
- }
-
- /**
* @return the context
*/
public Context getContext() {
@@ -499,12 +491,12 @@ public class ParseContext {
this.queryProperties = queryProperties;
}
- public TableDesc getFetchTabledesc() {
- return fetchTabledesc;
+ public TableDesc getFetchTableDesc() {
+ return fetchTableDesc;
}
- public void setFetchTabledesc(TableDesc fetchTabledesc) {
- this.fetchTabledesc = fetchTabledesc;
+ public void setFetchTabledesc(TableDesc fetchTableDesc) {
+ this.fetchTableDesc = fetchTableDesc;
}
public Operator<?> getFetchSource() {
@@ -522,4 +514,21 @@ public class ParseContext {
public void setFetchSink(ListSinkOperator fetchSink) {
this.fetchSink = fetchSink;
}
+
+ public AnalyzeRewriteContext getAnalyzeRewrite() {
+ return this.analyzeRewrite;
+ }
+
+ public void setAnalyzeRewrite(AnalyzeRewriteContext analyzeRewrite) {
+ this.analyzeRewrite = analyzeRewrite;
+ }
+
+ public CreateTableDesc getCreateTable() {
+ return this.createTableDesc;
+ }
+
+ public void setCreateTable(CreateTableDesc createTableDesc) {
+ this.createTableDesc = createTableDesc;
+ }
+
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java Tue Mar 10 04:20:45 2015
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.parse;
-import java.lang.StringBuffer;
import java.util.List;
import java.util.Set;
import java.util.Stack;
@@ -41,8 +40,8 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.StatsNoJobWork;
-import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.hive.ql.plan.StatsWork;
+import org.apache.hadoop.hive.ql.plan.TezWork;
import org.apache.hadoop.mapred.InputFormat;
/**
@@ -77,10 +76,8 @@ public class ProcessAnalyzeTable impleme
ParseContext parseContext = context.parseContext;
Class<? extends InputFormat> inputFormat = tableScan.getConf().getTableMetadata()
.getInputFormatClass();
- QB queryBlock = parseContext.getQB();
- QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
-
- if (parseInfo.isAnalyzeCommand()) {
+
+ if (parseContext.getQueryProperties().isAnalyzeCommand()) {
assert tableScan.getChildOperators() == null
|| tableScan.getChildOperators().size() == 0;
@@ -102,14 +99,14 @@ public class ProcessAnalyzeTable impleme
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
// There will not be any Tez job above this task
- StatsNoJobWork snjWork = new StatsNoJobWork(parseContext.getQB().getParseInfo().getTableSpec());
+ StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec());
snjWork.setStatsReliable(parseContext.getConf().getBoolVar(
HiveConf.ConfVars.HIVE_STATS_RELIABLE));
// If partition is specified, get pruned partition list
- Set<Partition> confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+ Set<Partition> confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan);
if (confirmedParts.size() > 0) {
- Table source = parseContext.getQB().getMetaData().getTableForAlias(alias);
- List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+ Table source = tableScan.getConf().getTableMetadata();
+ List<String> partCols = GenMapRedUtils.getPartitionColumns(tableScan);
PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts,
partCols, false);
snjWork.setPrunedPartitionList(partList);
@@ -124,8 +121,8 @@ public class ProcessAnalyzeTable impleme
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS;
// The plan consists of a simple TezTask followed by a StatsTask.
// The Tez task is just a simple TableScanOperator
-
- StatsWork statsWork = new StatsWork(parseInfo.getTableSpec());
+
+ StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec());
statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix());
statsWork.setSourceTask(context.currentTask);
statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
@@ -134,7 +131,7 @@ public class ProcessAnalyzeTable impleme
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
// The plan consists of a StatsTask only.
- if (parseInfo.isNoScanAnalyzeCommand()) {
+ if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) {
statsTask.setParentTasks(null);
statsWork.setNoScanAnalyzeCommand(true);
context.rootTasks.remove(context.currentTask);
@@ -142,17 +139,17 @@ public class ProcessAnalyzeTable impleme
}
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
- if (parseInfo.isPartialScanAnalyzeCommand()) {
- handlePartialScanCommand(tableScan, parseContext, parseInfo, statsWork, context, statsTask);
+ if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) {
+ handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask);
}
// NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list,
// and pass it to setTaskPlan as the last parameter
- Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+ Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan);
PrunedPartitionList partitions = null;
if (confirmedPartns.size() > 0) {
- Table source = queryBlock.getMetaData().getTableForAlias(alias);
- List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+ Table source = tableScan.getConf().getTableMetadata();
+ List<String> partCols = GenMapRedUtils.getPartitionColumns(tableScan);
partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false);
}
@@ -172,14 +169,14 @@ public class ProcessAnalyzeTable impleme
* It is composed of PartialScanTask followed by StatsTask.
*/
private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
- QBParseInfo parseInfo, StatsWork statsWork, GenTezProcContext context,
- Task<StatsWork> statsTask) throws SemanticException {
+ StatsWork statsWork, GenTezProcContext context, Task<StatsWork> statsTask)
+ throws SemanticException {
String aggregationKey = tableScan.getConf().getStatsAggPrefix();
StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
- List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(parseInfo, aggregationKeyBuffer);
+ List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
aggregationKey = aggregationKeyBuffer.toString();
-
+
// scan work
PartialScanWork scanWork = new PartialScanWork(inputPaths);
scanWork.setMapperCannotSpanPartns(true);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java Tue Mar 10 04:20:45 2015
@@ -30,7 +30,8 @@ import java.util.Set;
import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
/**
* Implementation of the parse information related to a query block.
@@ -69,12 +70,9 @@ public class QBParseInfo {
private boolean isPartialScanAnalyzeCommand; // used for the analyze command (statistics)
// (partialscan)
- private final HashMap<String, tableSpec> tableSpecs; // used for statistics
+ private final HashMap<String, TableSpec> tableSpecs; // used for statistics
- private String tableName; // used for column statistics
- private List<String> colName; // used for column statistics
- private List<String> colType; // used for column statistics
- private boolean isTblLvl; // used for column statistics
+ private AnalyzeRewriteContext analyzeRewrite;
/**
@@ -147,7 +145,7 @@ public class QBParseInfo {
aliasToLateralViews = new HashMap<String, ArrayList<ASTNode>>();
- tableSpecs = new HashMap<String, BaseSemanticAnalyzer.tableSpec>();
+ tableSpecs = new HashMap<String, BaseSemanticAnalyzer.TableSpec>();
}
@@ -560,18 +558,18 @@ public class QBParseInfo {
return isInsertToTable;
}
- public void addTableSpec(String tName, tableSpec tSpec) {
+ public void addTableSpec(String tName, TableSpec tSpec) {
tableSpecs.put(tName, tSpec);
}
- public tableSpec getTableSpec(String tName) {
+ public TableSpec getTableSpec(String tName) {
return tableSpecs.get(tName);
}
/**
* This method is used only for the analyze command to get the partition specs
*/
- public tableSpec getTableSpec() {
+ public TableSpec getTableSpec() {
Iterator<String> tName = tableSpecs.keySet().iterator();
return tableSpecs.get(tName.next());
@@ -604,36 +602,12 @@ public class QBParseInfo {
SORT_BY_CLAUSE
}
- public String getTableName() {
- return tableName;
+ public AnalyzeRewriteContext getAnalyzeRewrite() {
+ return analyzeRewrite;
}
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public List<String> getColName() {
- return colName;
- }
-
- public void setColName(List<String> colName) {
- this.colName = colName;
- }
-
- public boolean isTblLvl() {
- return isTblLvl;
- }
-
- public void setTblLvl(boolean isTblLvl) {
- this.isTblLvl = isTblLvl;
- }
-
- public List<String> getColType() {
- return colType;
- }
-
- public void setColType(List<String> colType) {
- this.colType = colType;
+ public void setAnalyzeRewrite(AnalyzeRewriteContext analyzeRewrite) {
+ this.analyzeRewrite = analyzeRewrite;
}
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -116,7 +116,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.optimizer.Optimizer;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
@@ -290,6 +290,9 @@ public class SemanticAnalyzer extends Ba
/** Not thread-safe. */
final ASTSearcher astSearcher = new ASTSearcher();
+
+ protected AnalyzeRewriteContext analyzeRewrite;
+ private CreateTableDesc tableDesc;
static class Phase1Ctx {
String dest;
@@ -388,21 +391,22 @@ public class SemanticAnalyzer extends Ba
idToTableNameMap = pctx.getIdToTableNameMap();
uCtx = pctx.getUCtx();
listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer();
- qb = pctx.getQB();
prunedPartitions = pctx.getPrunedPartitions();
fetchTask = pctx.getFetchTask();
setLineageInfo(pctx.getLineageInfo());
}
public ParseContext getParseContext() {
- return new ParseContext(conf, qb, opToPartPruner, opToPartList, topOps,
+ // Make sure the basic query properties are initialized
+ copyInfoToQueryProperties(queryProperties);
+ return new ParseContext(conf, opToPartPruner, opToPartList, topOps,
new HashSet<JoinOperator>(joinContext.keySet()),
new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()),
loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
listMapJoinOpsNoReducer, prunedPartitions,
opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
- opToPartToSkewedPruner, viewAliasToInput,
- reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties);
+ opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
+ analyzeRewrite, tableDesc, queryProperties);
}
@SuppressWarnings("nls")
@@ -645,12 +649,11 @@ public class SemanticAnalyzer extends Ba
(ASTNode) tabref.getChild(0),
ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
}
- qb.getParseInfo().setTabSample(
- alias,
- new TableSample(
+ TableSample tabSample = new TableSample(
unescapeIdentifier(sampleClause.getChild(0).getText()),
unescapeIdentifier(sampleClause.getChild(1).getText()),
- sampleCols));
+ sampleCols);
+ qb.getParseInfo().setTabSample(alias, tabSample);
if (unparseTranslator.isEnabled()) {
for (ASTNode sampleCol : sampleCols) {
unparseTranslator.addIdentifierTranslation((ASTNode) sampleCol
@@ -1631,7 +1634,7 @@ public class SemanticAnalyzer extends Ba
if (qb.getParseInfo().isAnalyzeCommand()) {
// allow partial partition specification for nonscan since noscan is fast.
- tableSpec ts = new tableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan);
+ TableSpec ts = new TableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan);
if (ts.specType == SpecType.DYNAMIC_PARTITION) { // dynamic partitions
try {
ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec);
@@ -1663,6 +1666,7 @@ public class SemanticAnalyzer extends Ba
}
}
+ tab.setTableSpec(ts);
qb.getParseInfo().addTableSpec(alias, ts);
}
@@ -1704,7 +1708,7 @@ public class SemanticAnalyzer extends Ba
ASTNode ast = qbp.getDestForClause(name);
switch (ast.getToken().getType()) {
case HiveParser.TOK_TAB: {
- tableSpec ts = new tableSpec(db, conf, ast);
+ TableSpec ts = new TableSpec(db, conf, ast);
if (ts.tableHandle.isView()) {
throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
}
@@ -1716,7 +1720,7 @@ public class SemanticAnalyzer extends Ba
.getMsg(ast, "The class is " + outputFormatClass.toString()));
}
- // tableSpec ts is got from the query (user specified),
+ // TableSpec ts is got from the query (user specified),
// which means the user didn't specify partitions in their query,
// but whether the table itself is partitioned is not know.
if (ts.specType != SpecType.STATIC_PARTITION) {
@@ -1769,7 +1773,7 @@ public class SemanticAnalyzer extends Ba
"Error creating temporary folder on: " + location.toString()), e);
}
if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
- tableSpec ts = new tableSpec(db, conf, this.ast);
+ TableSpec ts = new TableSpec(db, conf, this.ast);
// Set that variable to automatically collect stats during the MapReduce job
qb.getParseInfo().setIsInsertToTable(true);
// Add the table spec for the destination table.
@@ -9328,9 +9332,11 @@ public class SemanticAnalyzer extends Ba
}
// check if this table is sampled and needs more than input pruning
- Operator<? extends OperatorDesc> tableOp = top;
+ Operator<? extends OperatorDesc> op = top;
TableSample ts = qb.getParseInfo().getTabSample(alias);
if (ts != null) {
+ TableScanOperator tableScanOp = (TableScanOperator) top;
+ tableScanOp.getConf().setTableSample(ts);
int num = ts.getNumerator();
int den = ts.getDenominator();
ArrayList<ASTNode> sampleExprs = ts.getExprs();
@@ -9389,7 +9395,7 @@ public class SemanticAnalyzer extends Ba
LOG.info("No need for sample filter");
ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols,
colsEqual, alias, rwsch, qb.getMetaData(), null);
- tableOp = OperatorFactory.getAndMakeChild(new FilterDesc(
+ op = OperatorFactory.getAndMakeChild(new FilterDesc(
samplePredicate, true, new SampleDesc(ts.getNumerator(), ts
.getDenominator(), tabBucketCols, true)),
new RowSchema(rwsch.getColumnInfos()), top);
@@ -9399,7 +9405,7 @@ public class SemanticAnalyzer extends Ba
LOG.info("Need sample filter");
ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols,
colsEqual, alias, rwsch, qb.getMetaData(), null);
- tableOp = OperatorFactory.getAndMakeChild(new FilterDesc(
+ op = OperatorFactory.getAndMakeChild(new FilterDesc(
samplePredicate, true),
new RowSchema(rwsch.getColumnInfos()), top);
}
@@ -9429,7 +9435,7 @@ public class SemanticAnalyzer extends Ba
qb.getParseInfo().setTabSample(alias, tsSample);
ExprNodeDesc samplePred = genSamplePredicate(tsSample, tab
.getBucketCols(), true, alias, rwsch, qb.getMetaData(), null);
- tableOp = OperatorFactory
+ op = OperatorFactory
.getAndMakeChild(new FilterDesc(samplePred, true,
new SampleDesc(tsSample.getNumerator(), tsSample
.getDenominator(), tab.getBucketCols(), true)),
@@ -9447,7 +9453,7 @@ public class SemanticAnalyzer extends Ba
.valueOf(460476415)));
ExprNodeDesc samplePred = genSamplePredicate(tsSample, null, false,
alias, rwsch, qb.getMetaData(), randFunc);
- tableOp = OperatorFactory.getAndMakeChild(new FilterDesc(
+ op = OperatorFactory.getAndMakeChild(new FilterDesc(
samplePred, true),
new RowSchema(rwsch.getColumnInfos()), top);
}
@@ -9455,10 +9461,10 @@ public class SemanticAnalyzer extends Ba
}
}
- Operator output = putOpInsertMap(tableOp, rwsch);
+ Operator output = putOpInsertMap(op, rwsch);
if (LOG.isDebugEnabled()) {
- LOG.debug("Created Table Plan for " + alias + " " + tableOp.toString());
+ LOG.debug("Created Table Plan for " + alias + " " + op.toString());
}
return output;
@@ -9503,7 +9509,7 @@ public class SemanticAnalyzer extends Ba
tsDesc.addVirtualCols(vcList);
String tblName = tab.getTableName();
- tableSpec tblSpec = qbp.getTableSpec(alias);
+ TableSpec tblSpec = qbp.getTableSpec(alias);
Map<String, String> partSpec = tblSpec.getPartSpec();
if (partSpec != null) {
@@ -10014,14 +10020,15 @@ public class SemanticAnalyzer extends Ba
}
// 4. Generate Parse Context for Optimizer & Physical compiler
- ParseContext pCtx = new ParseContext(conf, qb,
- opToPartPruner, opToPartList, topOps,
+ copyInfoToQueryProperties(queryProperties);
+ ParseContext pCtx = new ParseContext(conf, opToPartPruner, opToPartList, topOps,
new HashSet<JoinOperator>(joinContext.keySet()),
new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()),
loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner,
globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner,
- viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties);
+ viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
+ analyzeRewrite, tableDesc, queryProperties);
// 5. Take care of view creation
if (createVwDesc != null) {
@@ -10792,13 +10799,12 @@ public class SemanticAnalyzer extends Ba
}
// Handle different types of CREATE TABLE command
- CreateTableDesc crtTblDesc = null;
switch (command_type) {
case CREATE_TABLE: // REGULAR CREATE TABLE DDL
tblProps = addDefaultProperties(tblProps);
- crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols,
+ CreateTableDesc crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols,
bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
rowFormatParams.fieldEscape,
rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim,
@@ -10850,16 +10856,16 @@ public class SemanticAnalyzer extends Ba
tblProps = addDefaultProperties(tblProps);
- crtTblDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
+ tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim,
rowFormatParams.lineDelim, comment, storageFormat.getInputFormat(),
storageFormat.getOutputFormat(), location, storageFormat.getSerde(),
storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists,
skewedColNames, skewedValues);
- crtTblDesc.setStoredAsSubDirectories(storedAsDirs);
- crtTblDesc.setNullFormat(rowFormatParams.nullFormat);
- qb.setTableDesc(crtTblDesc);
+ tableDesc.setStoredAsSubDirectories(storedAsDirs);
+ tableDesc.setNullFormat(rowFormatParams.nullFormat);
+ qb.setTableDesc(tableDesc);
SessionState.get().setCommandType(HiveOperation.CREATETABLE_AS_SELECT);
@@ -12075,4 +12081,19 @@ public class SemanticAnalyzer extends Ba
selexpr.addChild(dot);
return selexpr;
}
+
+ private void copyInfoToQueryProperties(QueryProperties queryProperties) {
+ if (qb != null) {
+ queryProperties.setQuery(qb.getIsQuery());
+ queryProperties.setAnalyzeCommand(qb.getParseInfo().isAnalyzeCommand());
+ queryProperties.setPartialScanAnalyzeCommand(qb.getParseInfo().isPartialScanAnalyzeCommand());
+ queryProperties.setNoScanAnalyzeCommand(qb.getParseInfo().isNoScanAnalyzeCommand());
+ queryProperties.setAnalyzeRewrite(qb.isAnalyzeRewrite());
+ queryProperties.setCTAS(qb.getTableDesc() != null);
+ queryProperties.setInsertToTable(qb.getParseInfo().isInsertToTable());
+ queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() &&
+ !qb.getParseInfo().getDestToOrderBy().isEmpty());
+ queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit());
+ }
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java Tue Mar 10 04:20:45 2015
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.hooks.W
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc;
import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
@@ -86,13 +87,13 @@ public abstract class TaskCompiler {
Context ctx = pCtx.getContext();
GlobalLimitCtx globalLimitCtx = pCtx.getGlobalLimitCtx();
- QB qb = pCtx.getQB();
List<Task<MoveWork>> mvTask = new ArrayList<Task<MoveWork>>();
List<LoadTableDesc> loadTableWork = pCtx.getLoadTableWork();
List<LoadFileDesc> loadFileWork = pCtx.getLoadFileWork();
- boolean isCStats = qb.isAnalyzeRewrite();
+ boolean isCStats = pCtx.getQueryProperties().isAnalyzeRewrite();
+ int outerQueryLimit = pCtx.getQueryProperties().getOuterQueryLimit();
if (pCtx.getFetchTask() != null) {
return;
@@ -105,7 +106,7 @@ public abstract class TaskCompiler {
* If the select is from analyze table column rewrite, don't create a fetch task. Instead create
* a column stats task later.
*/
- if (pCtx.getQB().getIsQuery() && !isCStats) {
+ if (pCtx.getQueryProperties().isQuery() && !isCStats) {
if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
}
@@ -115,14 +116,13 @@ public abstract class TaskCompiler {
String cols = loadFileDesc.getColumns();
String colTypes = loadFileDesc.getColumnTypes();
- TableDesc resultTab = pCtx.getFetchTabledesc();
+ TableDesc resultTab = pCtx.getFetchTableDesc();
if (resultTab == null) {
String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
}
- FetchWork fetch = new FetchWork(loadFileDesc.getSourcePath(),
- resultTab, qb.getParseInfo().getOuterQueryLimit());
+ FetchWork fetch = new FetchWork(loadFileDesc.getSourcePath(), resultTab, outerQueryLimit);
fetch.setSource(pCtx.getFetchSource());
fetch.setSink(pCtx.getFetchSink());
@@ -138,7 +138,7 @@ public abstract class TaskCompiler {
globalLimitCtx.disableOpt();
}
- if (qb.getParseInfo().getOuterQueryLimit() == 0) {
+ if (outerQueryLimit == 0) {
// Believe it or not, some tools do generate queries with limit 0 and than expect
// query to run quickly. Lets meet their requirement.
LOG.info("Limit 0. No query execution needed.");
@@ -166,17 +166,18 @@ public abstract class TaskCompiler {
boolean oneLoadFile = true;
for (LoadFileDesc lfd : loadFileWork) {
- if (qb.isCTAS()) {
+ if (pCtx.getQueryProperties().isCTAS()) {
assert (oneLoadFile); // should not have more than 1 load file for
// CTAS
// make the movetask's destination directory the table's destination.
Path location;
- String loc = qb.getTableDesc().getLocation();
+ String loc = pCtx.getCreateTable().getLocation();
if (loc == null) {
// get the table's default location
Path targetPath;
try {
- String[] names = Utilities.getDbTableName(qb.getTableDesc().getTableName());
+ String[] names = Utilities.getDbTableName(
+ pCtx.getCreateTable().getTableName());
if (!db.databaseExists(names[0])) {
throw new SemanticException("ERROR: The database " + names[0]
+ " does not exist.");
@@ -208,7 +209,8 @@ public abstract class TaskCompiler {
* a column stats task instead of a fetch task to persist stats to the metastore.
*/
if (isCStats) {
- genColumnStatsTask(qb, loadTableWork, loadFileWork, rootTasks);
+ genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadTableWork, loadFileWork,
+ rootTasks, outerQueryLimit);
}
// For each task, set the key descriptor for the reducer
@@ -226,9 +228,9 @@ public abstract class TaskCompiler {
decideExecMode(rootTasks, ctx, globalLimitCtx);
- if (qb.isCTAS()) {
+ if (pCtx.getQueryProperties().isCTAS()) {
// generate a DDL task and make it a dependent task of the leaf
- CreateTableDesc crtTblDesc = qb.getTableDesc();
+ CreateTableDesc crtTblDesc = pCtx.getCreateTable();
crtTblDesc.validate(conf);
@@ -305,16 +307,15 @@ public abstract class TaskCompiler {
* @param qb
*/
@SuppressWarnings("unchecked")
- protected void genColumnStatsTask(QB qb, List<LoadTableDesc> loadTableWork,
- List<LoadFileDesc> loadFileWork, List<Task<? extends Serializable>> rootTasks) {
- QBParseInfo qbParseInfo = qb.getParseInfo();
+ protected void genColumnStatsTask(AnalyzeRewriteContext analyzeRewrite, List<LoadTableDesc> loadTableWork,
+ List<LoadFileDesc> loadFileWork, List<Task<? extends Serializable>> rootTasks, int outerQueryLimit) {
ColumnStatsTask cStatsTask = null;
ColumnStatsWork cStatsWork = null;
FetchWork fetch = null;
- String tableName = qbParseInfo.getTableName();
- List<String> colName = qbParseInfo.getColName();
- List<String> colType = qbParseInfo.getColType();
- boolean isTblLevel = qbParseInfo.isTblLvl();
+ String tableName = analyzeRewrite.getTableName();
+ List<String> colName = analyzeRewrite.getColName();
+ List<String> colType = analyzeRewrite.getColType();
+ boolean isTblLevel = analyzeRewrite.isTblLvl();
String cols = loadFileWork.get(0).getColumns();
String colTypes = loadFileWork.get(0).getColumnTypes();
@@ -322,8 +323,7 @@ public abstract class TaskCompiler {
String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
- fetch = new FetchWork(loadFileWork.get(0).getSourcePath(),
- resultTab, qb.getParseInfo().getOuterQueryLimit());
+ fetch = new FetchWork(loadFileWork.get(0).getSourcePath(), resultTab, outerQueryLimit);
ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName,
colName, colType, isTblLevel);
@@ -390,7 +390,6 @@ public abstract class TaskCompiler {
*/
public ParseContext getParseContext(ParseContext pCtx, List<Task<? extends Serializable>> rootTasks) {
ParseContext clone = new ParseContext(conf,
- pCtx.getQB(),
pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(),
pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(),
pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getContext(),
@@ -400,7 +399,7 @@ public abstract class TaskCompiler {
pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks,
pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(),
pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(),
- pCtx.getQueryProperties());
+ pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getQueryProperties());
clone.setFetchTask(pCtx.getFetchTask());
clone.setLineageInfo(pCtx.getLineageInfo());
clone.setMapJoinOps(pCtx.getMapJoinOps());
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Mar 10 04:20:45 2015
@@ -555,7 +555,7 @@ public class TypeCheckProcFactory {
private static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) {
ObjectInspector inspector = colInfo.getObjectInspector();
- if (inspector instanceof ConstantObjectInspector &&
+ if (inspector instanceof ConstantObjectInspector &&
inspector instanceof PrimitiveObjectInspector) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
@@ -994,8 +994,15 @@ public class TypeCheckProcFactory {
.getText());
// NOTE: tableAlias must be a valid non-ambiguous table alias,
// because we've checked that in TOK_TABLE_OR_COL's process method.
- ColumnInfo colInfo = input.get(tableAlias, ((ExprNodeConstantDesc) nodeOutputs[1]).getValue()
- .toString());
+ String colName;
+ if (nodeOutputs[1] instanceof ExprNodeConstantDesc) {
+ colName = ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString();
+ } else if (nodeOutputs[1] instanceof ExprNodeColumnDesc) {
+ colName = ((ExprNodeColumnDesc)nodeOutputs[1]).getColumn();
+ } else {
+ throw new SemanticException("Unexpected ExprNode : " + nodeOutputs[1]);
+ }
+ ColumnInfo colInfo = input.get(tableAlias, colName);
if (colInfo == null) {
ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java Tue Mar 10 04:20:45 2015
@@ -40,8 +40,6 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-import org.apache.hadoop.hive.ql.parse.QB;
-import org.apache.hadoop.hive.ql.parse.QBParseInfo;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.SparkWork;
@@ -84,10 +82,8 @@ public class SparkProcessAnalyzeTable im
@SuppressWarnings("rawtypes")
Class<? extends InputFormat> inputFormat = tableScan.getConf().getTableMetadata()
.getInputFormatClass();
- QB queryBlock = parseContext.getQB();
- QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
- if (parseInfo.isAnalyzeCommand()) {
+ if (parseContext.getQueryProperties().isAnalyzeCommand()) {
Preconditions.checkArgument(tableScan.getChildOperators() == null
|| tableScan.getChildOperators().size() == 0,
"AssertionError: expected tableScan.getChildOperators() to be null, "
@@ -102,14 +98,14 @@ public class SparkProcessAnalyzeTable im
Preconditions.checkArgument(alias != null, "AssertionError: expected alias to be not null");
SparkWork sparkWork = context.currentTask.getWork();
- boolean partialScan = parseInfo.isPartialScanAnalyzeCommand();
- boolean noScan = parseInfo.isNoScanAnalyzeCommand();
+ boolean partialScan = parseContext.getQueryProperties().isPartialScanAnalyzeCommand();
+ boolean noScan = parseContext.getQueryProperties().isNoScanAnalyzeCommand();
if (inputFormat.equals(OrcInputFormat.class) && (noScan || partialScan)) {
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
// There will not be any Spark job above this task
- StatsNoJobWork snjWork = new StatsNoJobWork(parseContext.getQB().getParseInfo().getTableSpec());
+ StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec());
snjWork.setStatsReliable(parseContext.getConf().getBoolVar(
HiveConf.ConfVars.HIVE_STATS_RELIABLE));
Task<StatsNoJobWork> snjTask = TaskFactory.get(snjWork, parseContext.getConf());
@@ -123,7 +119,7 @@ public class SparkProcessAnalyzeTable im
// The plan consists of a simple SparkTask followed by a StatsTask.
// The Spark task is just a simple TableScanOperator
- StatsWork statsWork = new StatsWork(parseInfo.getTableSpec());
+ StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec());
statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix());
statsWork.setSourceTask(context.currentTask);
statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
@@ -132,7 +128,7 @@ public class SparkProcessAnalyzeTable im
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
// The plan consists of a StatsTask only.
- if (parseInfo.isNoScanAnalyzeCommand()) {
+ if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) {
statsTask.setParentTasks(null);
statsWork.setNoScanAnalyzeCommand(true);
context.rootTasks.remove(context.currentTask);
@@ -140,17 +136,17 @@ public class SparkProcessAnalyzeTable im
}
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
- if (parseInfo.isPartialScanAnalyzeCommand()) {
- handlePartialScanCommand(tableScan, parseContext, parseInfo, statsWork, context, statsTask);
+ if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) {
+ handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask);
}
// NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list,
// and pass it to setTaskPlan as the last parameter
- Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+ Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan);
PrunedPartitionList partitions = null;
if (confirmedPartns.size() > 0) {
- Table source = queryBlock.getMetaData().getTableForAlias(alias);
- List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+ Table source = tableScan.getConf().getTableMetadata();
+ List<String> partCols = GenMapRedUtils.getPartitionColumns(tableScan);
partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false);
}
@@ -169,11 +165,11 @@ public class SparkProcessAnalyzeTable im
* It is composed of PartialScanTask followed by StatsTask.
*/
private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
- QBParseInfo parseInfo, StatsWork statsWork, GenSparkProcContext context,
- Task<StatsWork> statsTask) throws SemanticException {
+ StatsWork statsWork, GenSparkProcContext context, Task<StatsWork> statsTask)
+ throws SemanticException {
String aggregationKey = tableScan.getConf().getStatsAggPrefix();
StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
- List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(parseInfo, aggregationKeyBuffer);
+ List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
aggregationKey = aggregationKeyBuffer.toString();
// scan work
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java Tue Mar 10 04:20:45 2015
@@ -72,7 +72,7 @@ public class DropTableDesc extends DDLDe
}
public DropTableDesc(String tableName, Map<Integer, List<ExprNodeGenericFuncDesc>> partSpecs,
- boolean expectView, boolean ignoreProtection) {
+ boolean expectView, boolean ignoreProtection, boolean ifPurge) {
this.tableName = tableName;
this.partSpecs = new ArrayList<PartSpec>(partSpecs.size());
for (Map.Entry<Integer, List<ExprNodeGenericFuncDesc>> partSpec : partSpecs.entrySet()) {
@@ -83,6 +83,7 @@ public class DropTableDesc extends DDLDe
}
this.ignoreProtection = ignoreProtection;
this.expectView = expectView;
+ this.ifPurge = ifPurge;
}
/**
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Tue Mar 10 04:20:45 2015
@@ -32,6 +32,10 @@ import org.apache.hadoop.hive.ql.udf.gen
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.util.ReflectionUtils;
@@ -427,4 +431,17 @@ public class ExprNodeDescUtils {
}
return true;
}
+
+ public static PrimitiveTypeInfo deriveMinArgumentCast(
+ ExprNodeDesc childExpr, TypeInfo targetType) {
+ assert targetType instanceof PrimitiveTypeInfo : "Not a primitive type" + targetType;
+ PrimitiveTypeInfo pti = (PrimitiveTypeInfo)targetType;
+ // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
+ // We also don't do anything for non-primitive children (maybe we should assert).
+ if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
+ || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
+ PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
+ // If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
+ return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java Tue Mar 10 04:20:45 2015
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -80,7 +79,7 @@ public class MergeJoinWork extends BaseW
}
}
- @Explain(skipHeader=true, displayName = "Join")
+ @Explain(skipHeader = true, displayName = "Join")
public List<BaseWork> getBaseWorkList() {
return mergeWorkList;
}
@@ -89,7 +88,7 @@ public class MergeJoinWork extends BaseW
return ((MapWork) bigTableWork).getAliasToWork().keySet().iterator().next();
}
- @Explain(skipHeader=true, displayName = "Main")
+ @Explain(skipHeader = true, displayName = "Main")
public BaseWork getMainWork() {
return bigTableWork;
}
@@ -103,4 +102,9 @@ public class MergeJoinWork extends BaseW
public void addDummyOp(HashTableDummyOperator dummyOp) {
getMainWork().addDummyOp(dummyOp);
}
+
+ @Override
+ public List<HashTableDummyOperator> getDummyOps() {
+ return getMainWork().getDummyOps();
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java Tue Mar 10 04:20:45 2015
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -288,7 +288,7 @@ public class PTFDeserializer {
try {
@SuppressWarnings("unchecked")
Class<? extends TableFunctionResolver> rCls = (Class<? extends TableFunctionResolver>)
- Class.forName(className);
+ JavaUtils.loadClass(className);
return ReflectionUtils.newInstance(rCls, null);
} catch (Exception e) {
throw new HiveException(e);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Tue Mar 10 04:20:45 2015
@@ -31,6 +31,7 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -144,7 +145,7 @@ public final class PlanUtils {
serdeConstants.SERIALIZATION_LIB, localDirectoryDesc.getSerName());
}
if (localDirectoryDesc.getOutputFormat() != null){
- ret.setOutputFileFormatClass(Class.forName(localDirectoryDesc.getOutputFormat()));
+ ret.setOutputFileFormatClass(JavaUtils.loadClass(localDirectoryDesc.getOutputFormat()));
}
if (localDirectoryDesc.getNullFormat() != null) {
properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT,
@@ -307,7 +308,7 @@ public final class PlanUtils {
try {
if (crtTblDesc.getSerName() != null) {
- Class c = Class.forName(crtTblDesc.getSerName());
+ Class c = JavaUtils.loadClass(crtTblDesc.getSerName());
serdeClass = c;
}
@@ -356,8 +357,8 @@ public final class PlanUtils {
// replace the default input & output file format with those found in
// crtTblDesc
- Class c1 = Class.forName(crtTblDesc.getInputFormat());
- Class c2 = Class.forName(crtTblDesc.getOutputFormat());
+ Class c1 = JavaUtils.loadClass(crtTblDesc.getInputFormat());
+ Class c2 = JavaUtils.loadClass(crtTblDesc.getOutputFormat());
Class<? extends InputFormat> in_class = c1;
Class<? extends HiveOutputFormat> out_class = c2;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java Tue Mar 10 04:20:45 2015
@@ -20,9 +20,10 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
+
/**
* Client-side stats aggregator task.
*/
@@ -30,14 +31,14 @@ import org.apache.hadoop.hive.ql.parse.P
public class StatsNoJobWork implements Serializable {
private static final long serialVersionUID = 1L;
- private tableSpec tableSpecs;
+ private TableSpec tableSpecs;
private boolean statsReliable;
private PrunedPartitionList prunedPartitionList;
public StatsNoJobWork() {
}
- public StatsNoJobWork(tableSpec tableSpecs) {
+ public StatsNoJobWork(TableSpec tableSpecs) {
this.tableSpecs = tableSpecs;
}
@@ -45,7 +46,7 @@ public class StatsNoJobWork implements S
this.statsReliable = statsReliable;
}
- public tableSpec getTableSpecs() {
+ public TableSpec getTableSpecs() {
return tableSpecs;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java Tue Mar 10 04:20:45 2015
@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
/**
* ConditionalStats.
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.parse.B
public class StatsWork implements Serializable {
private static final long serialVersionUID = 1L;
- private tableSpec tableSpecs; // source table spec -- for TableScanOperator
+ private TableSpec tableSpecs; // source table spec -- for TableScanOperator
private LoadTableDesc loadTableDesc; // same as MoveWork.loadTableDesc -- for FileSinkOperator
private LoadFileDesc loadFileDesc; // same as MoveWork.loadFileDesc -- for FileSinkOperator
private String aggKey; // aggregation key prefix
@@ -58,7 +58,7 @@ public class StatsWork implements Serial
public StatsWork() {
}
- public StatsWork(tableSpec tableSpecs) {
+ public StatsWork(TableSpec tableSpecs) {
this.tableSpecs = tableSpecs;
}
@@ -74,7 +74,7 @@ public class StatsWork implements Serial
this.statsReliable = statsReliable;
}
- public tableSpec getTableSpecs() {
+ public TableSpec getTableSpecs() {
return tableSpecs;
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java Tue Mar 10 04:20:45 2015
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
+import org.apache.hadoop.hive.ql.parse.TableSample;
/**
* Table Scan Descriptor Currently, data is only read from a base source as part
@@ -91,9 +92,11 @@ public class TableScanDesc extends Abstr
// input file name (big) to bucket number
private Map<String, Integer> bucketFileNameMapping;
-
+
private boolean isMetadataOnly = false;
+ private transient TableSample tableSample;
+
private transient final Table tableMetadata;
@@ -268,4 +271,12 @@ public class TableScanDesc extends Abstr
public Table getTableMetadata() {
return tableMetadata;
}
+
+ public TableSample getTableSample() {
+ return tableSample;
+ }
+
+ public void setTableSample(TableSample tableSample) {
+ this.tableSample = tableSample;
+ }
}
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java Tue Mar 10 04:20:45 2015
@@ -19,9 +19,9 @@
package org.apache.hadoop.hive.ql.plan.ptf;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.plan.Explain;
-import org.apache.hadoop.util.StringUtils;
public abstract class PTFInputDef {
private String expressionTreeString;
@@ -43,7 +43,7 @@ public abstract class PTFInputDef {
@Explain(displayName = "output shape")
public String getOutputShapeExplain() {
RowSchema schema = outputShape.getRr().getRowSchema();
- return StringUtils.join(", ", schema.getSignature());
+ return StringUtils.join(schema.getSignature(), ", ");
}
public void setOutputShape(ShapeDetails outputShape) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Tue Mar 10 04:20:45 2015
@@ -588,8 +588,15 @@ public class SessionState {
Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true);
}
FsPermission currentHDFSDirPermission = fs.getFileStatus(rootHDFSDirPath).getPermission();
- LOG.debug("HDFS root scratch dir: " + rootHDFSDirPath + ", permission: "
- + currentHDFSDirPermission);
+ if (rootHDFSDirPath != null && rootHDFSDirPath.toUri() != null) {
+ String schema = rootHDFSDirPath.toUri().getScheme();
+ LOG.debug(
+ "HDFS root scratch dir: " + rootHDFSDirPath + " with schema " + schema + ", permission: " +
+ currentHDFSDirPermission);
+ } else {
+ LOG.debug(
+ "HDFS root scratch dir: " + rootHDFSDirPath + ", permission: " + currentHDFSDirPermission);
+ }
// If the root HDFS scratch dir already exists, make sure it is writeable.
if (!((currentHDFSDirPermission.toShort() & writableHDFSDirPermission
.toShort()) == writableHDFSDirPermission.toShort())) {
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Tue Mar 10 04:20:45 2015
@@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -69,7 +70,7 @@ public class JDBCStatsAggregator impleme
this.sourceTask = sourceTask;
try {
- Class.forName(driver).newInstance();
+ JavaUtils.loadClass(driver).newInstance();
} catch (Exception e) {
LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
return false;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Tue Mar 10 04:20:45 2015
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -69,7 +70,7 @@ public class JDBCStatsPublisher implemen
String driver = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER);
try {
- Class.forName(driver).newInstance();
+ JavaUtils.loadClass(driver).newInstance();
} catch (Exception e) {
LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
return false;
@@ -272,7 +273,7 @@ public class JDBCStatsPublisher implemen
this.hiveconf = hconf;
connectionString = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING);
String driver = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER);
- Class.forName(driver).newInstance();
+ JavaUtils.loadClass(driver).newInstance();
synchronized(DriverManager.class) {
DriverManager.setLoginTimeout(timeout);
conn = DriverManager.getConnection(connectionString);
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java Tue Mar 10 04:20:45 2015
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.BlockLocatio
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.common.ValidReadTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -647,7 +648,7 @@ public class CompactorMR {
private static <T> T instantiate(Class<T> classType, String classname) throws IOException {
T t = null;
try {
- Class c = Class.forName(classname);
+ Class c = JavaUtils.loadClass(classname);
Object o = c.newInstance();
if (classType.isAssignableFrom(o.getClass())) {
t = (T)o;
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java Tue Mar 10 04:20:45 2015
@@ -231,6 +231,13 @@ public class GenericUDAFnGrams implement
NGramAggBuf myagg = (NGramAggBuf) agg;
List<Text> partialNGrams = (List<Text>) loi.getList(partial);
int n = Integer.parseInt(partialNGrams.get(partialNGrams.size()-1).toString());
+
+ // A value of 0 for n indicates that the mapper processed data that does not meet
+ // filter criteria, so merge() should be NO-OP.
+ if (n == 0) {
+ return;
+ }
+
if(myagg.n > 0 && myagg.n != n) {
throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'"
+ ", which usually is caused by a non-constant expression. Found '"+n+"' and '"