You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/03/10 05:20:48 UTC

svn commit: r1665403 [4/7] - in /hive/branches/spark: ./ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/predicate/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/ an...

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java Tue Mar 10 04:20:45 2015
@@ -416,8 +416,7 @@ public class SortedDynPartitionOptimizer
       // should honor the ordering of records provided by ORDER BY in SELECT statement
       ReduceSinkOperator parentRSOp = OperatorUtils.findSingleOperatorUpstream(parent,
           ReduceSinkOperator.class);
-      boolean isOrderBy = parseCtx.getQB().getParseInfo().getDestToOrderBy().size() > 0;
-      if (parentRSOp != null && isOrderBy) {
+      if (parentRSOp != null && parseCtx.getQueryProperties().hasOuterOrderBy()) {
         String parentRSOpOrder = parentRSOp.getConf().getOrder();
         if (parentRSOpOrder != null && !parentRSOpOrder.isEmpty() && sortPositions.isEmpty()) {
           newKeyCols.addAll(parentRSOp.getConf().getKeyCols());

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java Tue Mar 10 04:20:45 2015
@@ -97,8 +97,8 @@ public class StatsOptimizer implements T
   @Override
   public ParseContext transform(ParseContext pctx) throws SemanticException {
 
-    if (pctx.getFetchTask() != null || !pctx.getQB().getIsQuery() ||
-        pctx.getQB().isAnalyzeRewrite() || pctx.getQB().isCTAS() ||
+    if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() ||
+        pctx.getQueryProperties().isAnalyzeRewrite() || pctx.getQueryProperties().isCTAS() ||
         pctx.getLoadFileWork().size() > 1 || !pctx.getLoadTableWork().isEmpty()) {
       return pctx;
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTBuilder.java Tue Mar 10 04:20:45 2015
@@ -213,7 +213,7 @@ class ASTBuilder {
     case TIMESTAMP: {
       val = literal.getValue();
       type = HiveParser.TOK_TIMESTAMPLITERAL;
-      DateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
+      DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
       val = df.format(((Calendar) val).getTime());
       val = "'" + val + "'";
     }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java Tue Mar 10 04:20:45 2015
@@ -54,10 +54,12 @@ import org.apache.hadoop.hive.ql.parse.S
 import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
 import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseNumeric;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
@@ -68,7 +70,10 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToVarchar;
 import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -152,8 +157,13 @@ public class RexNodeConverter {
     // TODO: 1) Expand to other functions as needed 2) What about types other than primitive.
     TypeInfo tgtDT = null;
     GenericUDF tgtUdf = func.getGenericUDF();
-    boolean isNumeric = tgtUdf instanceof GenericUDFBaseNumeric,
-        isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare;
+
+    boolean isNumeric = (tgtUdf instanceof GenericUDFBaseBinary
+        && func.getTypeInfo().getCategory() == Category.PRIMITIVE
+        && (PrimitiveGrouping.NUMERIC_GROUP == PrimitiveObjectInspectorUtils.getPrimitiveGrouping(
+            ((PrimitiveTypeInfo) func.getTypeInfo()).getPrimitiveCategory())));
+    boolean isCompare = !isNumeric && tgtUdf instanceof GenericUDFBaseCompare;
+
     if (isNumeric) {
       tgtDT = func.getTypeInfo();
 
@@ -175,8 +185,7 @@ public class RexNodeConverter {
         } else if (isNumeric) {
           // For numeric, we'll do minimum necessary cast - if we cast to the type
           // of expression, bad things will happen.
-          GenericUDFBaseNumeric numericUdf = (GenericUDFBaseNumeric)tgtUdf;
-          PrimitiveTypeInfo minArgType = numericUdf.deriveMinArgumentCast(childExpr, tgtDT);
+          PrimitiveTypeInfo minArgType = ExprNodeDescUtils.deriveMinArgumentCast(childExpr, tgtDT);
           tmpExprNode = ParseUtils.createConversionCast(childExpr, minArgType);
         } else {
           throw new AssertionError("Unexpected " + tgtDT + " - not a numeric op or compare");

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -698,10 +698,10 @@ public abstract class BaseSemanticAnalyz
   }
 
   /**
-   * tableSpec.
+   * TableSpec.
    *
    */
-  public static class tableSpec {
+  public static class TableSpec {
     public String tableName;
     public Table tableHandle;
     public Map<String, String> partSpec; // has to use LinkedHashMap to enforce order
@@ -711,12 +711,12 @@ public abstract class BaseSemanticAnalyz
     public static enum SpecType {TABLE_ONLY, STATIC_PARTITION, DYNAMIC_PARTITION};
     public SpecType specType;
 
-    public tableSpec(Hive db, HiveConf conf, ASTNode ast)
+    public TableSpec(Hive db, HiveConf conf, ASTNode ast)
         throws SemanticException {
       this(db, conf, ast, true, false);
     }
 
-    public tableSpec(Hive db, HiveConf conf, String tableName, Map<String, String> partSpec)
+    public TableSpec(Hive db, HiveConf conf, String tableName, Map<String, String> partSpec)
         throws HiveException {
       this.tableName = tableName;
       this.partSpec = partSpec;
@@ -730,7 +730,7 @@ public abstract class BaseSemanticAnalyz
       }
     }
 
-    public tableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartitionsSpec,
+    public TableSpec(Hive db, HiveConf conf, ASTNode ast, boolean allowDynamicPartitionsSpec,
         boolean allowPartialPartitionsSpec) throws SemanticException {
       assert (ast.getToken().getType() == HiveParser.TOK_TAB
           || ast.getToken().getType() == HiveParser.TOK_TABLE_PARTITION
@@ -869,6 +869,47 @@ public abstract class BaseSemanticAnalyz
     }
   }
 
+  public class AnalyzeRewriteContext {
+
+    private String tableName;
+    private List<String> colName;
+    private List<String> colType;
+    private boolean tblLvl;
+
+
+    public String getTableName() {
+      return tableName;
+    }
+
+    public void setTableName(String tableName) {
+      this.tableName = tableName;
+    }
+
+    public List<String> getColName() {
+      return colName;
+    }
+
+    public void setColName(List<String> colName) {
+      this.colName = colName;
+    }
+
+    public boolean isTblLvl() {
+      return tblLvl;
+    }
+
+    public void setTblLvl(boolean isTblLvl) {
+      this.tblLvl = isTblLvl;
+    }
+
+    public List<String> getColType() {
+      return colType;
+    }
+
+    public void setColType(List<String> colType) {
+      this.colType = colType;
+    }
+  }
+
   /**
    * Gets the lineage information.
    *

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Tue Mar 10 04:20:45 2015
@@ -718,7 +718,7 @@ public class CalcitePlanner extends Sema
       hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE);
       hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE);
       hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
-      hepPgmBldr.addRuleInstance(ProjectRemoveRule.NAME_CALC_INSTANCE);
+      hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
       hepPgmBldr.addRuleInstance(UnionMergeRule.INSTANCE);
 
       hepPgm = hepPgmBldr.build();
@@ -792,7 +792,7 @@ public class CalcitePlanner extends Sema
       RelFieldTrimmer fieldTrimmer = new RelFieldTrimmer(null, HiveProject.DEFAULT_PROJECT_FACTORY,
           HiveFilter.DEFAULT_FILTER_FACTORY, HiveJoin.HIVE_JOIN_FACTORY,
           RelFactories.DEFAULT_SEMI_JOIN_FACTORY, HiveSort.HIVE_SORT_REL_FACTORY,
-          HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY, true);
+          HiveAggregate.HIVE_AGGR_REL_FACTORY, HiveUnion.UNION_REL_FACTORY);
       basePlan = fieldTrimmer.trim(basePlan);
 
       // 5. Rerun PPD through Project as column pruning would have introduced DT

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnStatsSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -427,10 +427,12 @@ public class ColumnStatsSemanticAnalyzer
       qb = getQB();
       qb.setAnalyzeRewrite(true);
       qbp = qb.getParseInfo();
-      qbp.setTableName(tbl.getTableName());
-      qbp.setTblLvl(isTableLevel);
-      qbp.setColName(colNames);
-      qbp.setColType(colType);
+      analyzeRewrite = new AnalyzeRewriteContext();
+      analyzeRewrite.setTableName(tbl.getTableName());
+      analyzeRewrite.setTblLvl(isTableLevel);
+      analyzeRewrite.setColName(colNames);
+      analyzeRewrite.setColType(colType);
+      qbp.setAnalyzeRewrite(analyzeRewrite);
       initCtx(ctx);
       LOG.info("Invoking analyze on rewritten query");
       analyzeInternal(rewrittenTree);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -45,6 +45,7 @@ import org.antlr.runtime.tree.Tree;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -997,7 +998,7 @@ public class DDLSemanticAnalyzer extends
           StatsWork statDesc;
           if (oldTblPartLoc.equals(newTblPartLoc)) {
             // If we're merging to the same location, we can avoid some metastore calls
-            tableSpec tablepart = new tableSpec(this.db, conf, root);
+            TableSpec tablepart = new TableSpec(this.db, conf, root);
             statDesc = new StatsWork(tablepart);
           } else {
             statDesc = new StatsWork(ltd);
@@ -1036,7 +1037,7 @@ public class DDLSemanticAnalyzer extends
       typeName = indexType.getHandlerClsName();
     } else {
       try {
-        Class.forName(typeName);
+        JavaUtils.loadClass(typeName);
       } catch (Exception e) {
         throw new SemanticException("class name provided for index handler not found.", e);
       }
@@ -1618,7 +1619,7 @@ public class DDLSemanticAnalyzer extends
         StatsWork statDesc;
         if (oldTblPartLoc.equals(newTblPartLoc)) {
           // If we're merging to the same location, we can avoid some metastore calls
-          tableSpec tablepart = new tableSpec(db, conf, tableName, partSpec);
+          TableSpec tablepart = new TableSpec(db, conf, tableName, partSpec);
           statDesc = new StatsWork(tablepart);
         } else {
           statDesc = new StatsWork(ltd);
@@ -2626,6 +2627,7 @@ public class DDLSemanticAnalyzer extends
     // popular case but that's kinda hacky. Let's not do it for now.
     boolean canGroupExprs = ifExists;
 
+    boolean mustPurge = (ast.getFirstChildWithType(HiveParser.KW_PURGE) != null);
     Table tab = getTable(qualified);
     Map<Integer, List<ExprNodeGenericFuncDesc>> partSpecs =
         getFullPartitionSpecs(ast, tab, canGroupExprs);
@@ -2640,7 +2642,7 @@ public class DDLSemanticAnalyzer extends
     addTableDropPartsOutputs(tab, partSpecs.values(), !ifExists, ignoreProtection);
 
     DropTableDesc dropTblDesc =
-        new DropTableDesc(getDotName(qualified), partSpecs, expectView, ignoreProtection);
+        new DropTableDesc(getDotName(qualified), partSpecs, expectView, ignoreProtection, mustPurge);
     rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), dropTblDesc), conf));
   }
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -18,13 +18,10 @@
 
 package org.apache.hadoop.hive.ql.parse;
 
-import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
-
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Serializable;
 import java.net.URI;
-import java.util.List;
 
 import org.antlr.runtime.tree.Tree;
 import org.apache.hadoop.fs.FileStatus;
@@ -37,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.Ta
 import org.apache.hadoop.hive.ql.exec.TaskFactory;
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.PartitionIterable;
 import org.apache.hadoop.hive.ql.plan.CopyWork;
 
 /**
@@ -59,7 +57,7 @@ public class ExportSemanticAnalyzer exte
     URI toURI = EximUtil.getValidatedURI(conf, tmpPath);
 
     // initialize source table/partition
-    tableSpec ts = new tableSpec(db, conf, (ASTNode) tableTree, false, true);
+    TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree, false, true);
     EximUtil.validateTable(ts.tableHandle);
     try {
       FileSystem fs = FileSystem.get(toURI, conf);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Mar 10 04:20:45 2015
@@ -1118,8 +1118,8 @@ partitionLocation
 alterStatementSuffixDropPartitions[boolean table]
 @init { pushMsg("drop partition statement", state); }
 @after { popMsg(state); }
-    : KW_DROP ifExists? dropPartitionSpec (COMMA dropPartitionSpec)* ignoreProtection?
-    -> { table }? ^(TOK_ALTERTABLE_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection?)
+    : KW_DROP ifExists? dropPartitionSpec (COMMA dropPartitionSpec)* ignoreProtection? KW_PURGE?
+    -> { table }? ^(TOK_ALTERTABLE_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection? KW_PURGE?)
     ->            ^(TOK_ALTERVIEW_DROPPARTS dropPartitionSpec+ ifExists? ignoreProtection?)
     ;
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -199,7 +199,7 @@ public class LoadSemanticAnalyzer extend
     }
 
     // initialize destination table/partition
-    tableSpec ts = new tableSpec(db, conf, (ASTNode) tableTree);
+    TableSpec ts = new TableSpec(db, conf, (ASTNode) tableTree);
 
     if (ts.tableHandle.isOffline()){
       throw new SemanticException(

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java Tue Mar 10 04:20:45 2015
@@ -42,6 +42,8 @@ import org.apache.hadoop.hive.ql.hooks.L
 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
 import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
 import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
+import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
 import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.FilterDesc.SampleDesc;
 import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
@@ -61,7 +63,7 @@ import org.apache.hadoop.hive.ql.plan.Ta
  **/
 
 public class ParseContext {
-  private QB qb;
+
   private HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner;
   private HashMap<TableScanOperator, PrunedPartitionList> opToPartList;
   private HashMap<TableScanOperator, SampleDesc> opToSamplePruner;
@@ -98,10 +100,14 @@ public class ParseContext {
   private FetchTask fetchTask;
   private QueryProperties queryProperties;
 
-  private TableDesc fetchTabledesc;
+  private TableDesc fetchTableDesc;
   private Operator<?> fetchSource;
   private ListSinkOperator fetchSink;
 
+  private AnalyzeRewriteContext analyzeRewrite;
+  private CreateTableDesc createTableDesc;
+
+
   public ParseContext() {
   }
 
@@ -142,7 +148,6 @@ public class ParseContext {
    */
   public ParseContext(
       HiveConf conf,
-      QB qb,
       HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner,
       HashMap<TableScanOperator, PrunedPartitionList> opToPartList,
       HashMap<String, Operator<? extends OperatorDesc>> topOps,
@@ -159,9 +164,9 @@ public class ParseContext {
       Map<TableScanOperator, Map<String, ExprNodeDesc>> opToPartToSkewedPruner,
       Map<String, ReadEntity> viewAliasToInput,
       List<ReduceSinkOperator> reduceSinkOperatorsAddedByEnforceBucketingSorting,
+      AnalyzeRewriteContext analyzeRewrite, CreateTableDesc createTableDesc,
       QueryProperties queryProperties) {
     this.conf = conf;
-    this.qb = qb;
     this.opToPartPruner = opToPartPruner;
     this.opToPartList = opToPartList;
     this.joinOps = joinOps;
@@ -184,25 +189,12 @@ public class ParseContext {
     this.viewAliasToInput = viewAliasToInput;
     this.reduceSinkOperatorsAddedByEnforceBucketingSorting =
         reduceSinkOperatorsAddedByEnforceBucketingSorting;
+    this.analyzeRewrite = analyzeRewrite;
+    this.createTableDesc = createTableDesc;
     this.queryProperties = queryProperties;
   }
 
   /**
-   * @return the qb
-   */
-  public QB getQB() {
-    return qb;
-  }
-
-  /**
-   * @param qb
-   *          the qb to set
-   */
-  public void setQB(QB qb) {
-    this.qb = qb;
-  }
-
-  /**
    * @return the context
    */
   public Context getContext() {
@@ -499,12 +491,12 @@ public class ParseContext {
     this.queryProperties = queryProperties;
   }
 
-  public TableDesc getFetchTabledesc() {
-    return fetchTabledesc;
+  public TableDesc getFetchTableDesc() {
+    return fetchTableDesc;
   }
 
-  public void setFetchTabledesc(TableDesc fetchTabledesc) {
-    this.fetchTabledesc = fetchTabledesc;
+  public void setFetchTabledesc(TableDesc fetchTableDesc) {
+    this.fetchTableDesc = fetchTableDesc;
   }
 
   public Operator<?> getFetchSource() {
@@ -522,4 +514,21 @@ public class ParseContext {
   public void setFetchSink(ListSinkOperator fetchSink) {
     this.fetchSink = fetchSink;
   }
+
+  public AnalyzeRewriteContext getAnalyzeRewrite() {
+    return this.analyzeRewrite;
+  }
+
+  public void setAnalyzeRewrite(AnalyzeRewriteContext analyzeRewrite) {
+    this.analyzeRewrite = analyzeRewrite;
+  }
+
+  public CreateTableDesc getCreateTable() {
+    return this.createTableDesc;
+  }
+
+  public void setCreateTable(CreateTableDesc createTableDesc) {
+    this.createTableDesc = createTableDesc;
+  }
+
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java Tue Mar 10 04:20:45 2015
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hive.ql.parse;
 
-import java.lang.StringBuffer;
 import java.util.List;
 import java.util.Set;
 import java.util.Stack;
@@ -41,8 +40,8 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.StatsNoJobWork;
-import org.apache.hadoop.hive.ql.plan.TezWork;
 import org.apache.hadoop.hive.ql.plan.StatsWork;
+import org.apache.hadoop.hive.ql.plan.TezWork;
 import org.apache.hadoop.mapred.InputFormat;
 
 /**
@@ -77,10 +76,8 @@ public class ProcessAnalyzeTable impleme
     ParseContext parseContext = context.parseContext;
     Class<? extends InputFormat> inputFormat = tableScan.getConf().getTableMetadata()
         .getInputFormatClass();
-    QB queryBlock = parseContext.getQB();
-    QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
-    
-    if (parseInfo.isAnalyzeCommand()) {
+
+    if (parseContext.getQueryProperties().isAnalyzeCommand()) {
 
       assert tableScan.getChildOperators() == null
         || tableScan.getChildOperators().size() == 0;
@@ -102,14 +99,14 @@ public class ProcessAnalyzeTable impleme
         // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
 
         // There will not be any Tez job above this task
-        StatsNoJobWork snjWork = new StatsNoJobWork(parseContext.getQB().getParseInfo().getTableSpec());
+        StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec());
         snjWork.setStatsReliable(parseContext.getConf().getBoolVar(
             HiveConf.ConfVars.HIVE_STATS_RELIABLE));
         // If partition is specified, get pruned partition list
-        Set<Partition> confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+        Set<Partition> confirmedParts = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan);
         if (confirmedParts.size() > 0) {
-          Table source = parseContext.getQB().getMetaData().getTableForAlias(alias);
-          List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+          Table source = tableScan.getConf().getTableMetadata();
+          List<String> partCols = GenMapRedUtils.getPartitionColumns(tableScan);
           PrunedPartitionList partList = new PrunedPartitionList(source, confirmedParts,
               partCols, false);
           snjWork.setPrunedPartitionList(partList);
@@ -124,8 +121,8 @@ public class ProcessAnalyzeTable impleme
       // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS;
       // The plan consists of a simple TezTask followed by a StatsTask.
       // The Tez task is just a simple TableScanOperator
-      
-      StatsWork statsWork = new StatsWork(parseInfo.getTableSpec());
+
+      StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec());
       statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix());
       statsWork.setSourceTask(context.currentTask);
       statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
@@ -134,7 +131,7 @@ public class ProcessAnalyzeTable impleme
 
       // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
       // The plan consists of a StatsTask only.
-      if (parseInfo.isNoScanAnalyzeCommand()) {
+      if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) {
         statsTask.setParentTasks(null);
         statsWork.setNoScanAnalyzeCommand(true);
         context.rootTasks.remove(context.currentTask);
@@ -142,17 +139,17 @@ public class ProcessAnalyzeTable impleme
       }
 
       // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
-      if (parseInfo.isPartialScanAnalyzeCommand()) {
-        handlePartialScanCommand(tableScan, parseContext, parseInfo, statsWork, context, statsTask);
+      if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) {
+        handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask);
       }
 
       // NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list,
       // and pass it to setTaskPlan as the last parameter
-      Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+      Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan);
       PrunedPartitionList partitions = null;
       if (confirmedPartns.size() > 0) {
-        Table source = queryBlock.getMetaData().getTableForAlias(alias);
-        List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+        Table source = tableScan.getConf().getTableMetadata();
+        List<String> partCols = GenMapRedUtils.getPartitionColumns(tableScan);
         partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false);
       }
 
@@ -172,14 +169,14 @@ public class ProcessAnalyzeTable impleme
    * It is composed of PartialScanTask followed by StatsTask.
    */
   private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext, 
-      QBParseInfo parseInfo, StatsWork statsWork, GenTezProcContext context,
-      Task<StatsWork> statsTask) throws SemanticException {
+      StatsWork statsWork, GenTezProcContext context, Task<StatsWork> statsTask)
+              throws SemanticException {
 
     String aggregationKey = tableScan.getConf().getStatsAggPrefix();
     StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
-    List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(parseInfo, aggregationKeyBuffer);
+    List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
     aggregationKey = aggregationKeyBuffer.toString();
-    
+
     // scan work
     PartialScanWork scanWork = new PartialScanWork(inputPaths);
     scanWork.setMapperCannotSpanPartns(true);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java Tue Mar 10 04:20:45 2015
@@ -30,7 +30,8 @@ import java.util.Set;
 import org.antlr.runtime.tree.Tree;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
 
 /**
  * Implementation of the parse information related to a query block.
@@ -69,12 +70,9 @@ public class QBParseInfo {
   private boolean isPartialScanAnalyzeCommand; // used for the analyze command (statistics)
                                                // (partialscan)
 
-  private final HashMap<String, tableSpec> tableSpecs; // used for statistics
+  private final HashMap<String, TableSpec> tableSpecs; // used for statistics
 
-  private String tableName;   // used for column statistics
-  private List<String> colName;     // used for column statistics
-  private List<String> colType;    // used for column statistics
-  private boolean isTblLvl; // used for column statistics
+  private AnalyzeRewriteContext analyzeRewrite;
 
 
   /**
@@ -147,7 +145,7 @@ public class QBParseInfo {
 
     aliasToLateralViews = new HashMap<String, ArrayList<ASTNode>>();
 
-    tableSpecs = new HashMap<String, BaseSemanticAnalyzer.tableSpec>();
+    tableSpecs = new HashMap<String, BaseSemanticAnalyzer.TableSpec>();
 
   }
 
@@ -560,18 +558,18 @@ public class QBParseInfo {
     return isInsertToTable;
   }
 
-  public void addTableSpec(String tName, tableSpec tSpec) {
+  public void addTableSpec(String tName, TableSpec tSpec) {
     tableSpecs.put(tName, tSpec);
   }
 
-  public tableSpec getTableSpec(String tName) {
+  public TableSpec getTableSpec(String tName) {
     return tableSpecs.get(tName);
   }
 
   /**
    * This method is used only for the analyze command to get the partition specs
    */
-  public tableSpec getTableSpec() {
+  public TableSpec getTableSpec() {
 
     Iterator<String> tName = tableSpecs.keySet().iterator();
     return tableSpecs.get(tName.next());
@@ -604,36 +602,12 @@ public class QBParseInfo {
     SORT_BY_CLAUSE
   }
 
-  public String getTableName() {
-    return tableName;
+  public AnalyzeRewriteContext getAnalyzeRewrite() {
+    return analyzeRewrite;
   }
 
-  public void setTableName(String tableName) {
-    this.tableName = tableName;
-  }
-
-  public List<String> getColName() {
-    return colName;
-  }
-
-  public void setColName(List<String> colName) {
-    this.colName = colName;
-  }
-
-  public boolean isTblLvl() {
-    return isTblLvl;
-  }
-
-  public void setTblLvl(boolean isTblLvl) {
-    this.isTblLvl = isTblLvl;
-  }
-
-  public List<String> getColType() {
-    return colType;
-  }
-
-  public void setColType(List<String> colType) {
-    this.colType = colType;
+  public void setAnalyzeRewrite(AnalyzeRewriteContext analyzeRewrite) {
+    this.analyzeRewrite = analyzeRewrite;
   }
 
   /**

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Mar 10 04:20:45 2015
@@ -116,7 +116,7 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.optimizer.Optimizer;
 import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
 import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec.SpecType;
 import org.apache.hadoop.hive.ql.parse.CalcitePlanner.ASTSearcher;
 import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
 import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderSpec;
@@ -290,6 +290,9 @@ public class SemanticAnalyzer extends Ba
 
   /** Not thread-safe. */
   final ASTSearcher astSearcher = new ASTSearcher();
+  
+  protected AnalyzeRewriteContext analyzeRewrite;
+  private CreateTableDesc tableDesc;
 
   static class Phase1Ctx {
     String dest;
@@ -388,21 +391,22 @@ public class SemanticAnalyzer extends Ba
     idToTableNameMap = pctx.getIdToTableNameMap();
     uCtx = pctx.getUCtx();
     listMapJoinOpsNoReducer = pctx.getListMapJoinOpsNoReducer();
-    qb = pctx.getQB();
     prunedPartitions = pctx.getPrunedPartitions();
     fetchTask = pctx.getFetchTask();
     setLineageInfo(pctx.getLineageInfo());
   }
 
   public ParseContext getParseContext() {
-    return new ParseContext(conf, qb, opToPartPruner, opToPartList, topOps,
+    // Make sure the basic query properties are initialized
+    copyInfoToQueryProperties(queryProperties);
+    return new ParseContext(conf, opToPartPruner, opToPartList, topOps,
         new HashSet<JoinOperator>(joinContext.keySet()),
         new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()),
         loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
         listMapJoinOpsNoReducer, prunedPartitions,
         opToSamplePruner, globalLimitCtx, nameToSplitSample, inputs, rootTasks,
-        opToPartToSkewedPruner, viewAliasToInput,
-        reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties);
+        opToPartToSkewedPruner, viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
+        analyzeRewrite, tableDesc, queryProperties);
   }
 
   @SuppressWarnings("nls")
@@ -645,12 +649,11 @@ public class SemanticAnalyzer extends Ba
             (ASTNode) tabref.getChild(0),
             ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
       }
-      qb.getParseInfo().setTabSample(
-          alias,
-          new TableSample(
+      TableSample tabSample = new TableSample(
               unescapeIdentifier(sampleClause.getChild(0).getText()),
               unescapeIdentifier(sampleClause.getChild(1).getText()),
-              sampleCols));
+              sampleCols);
+      qb.getParseInfo().setTabSample(alias, tabSample);
       if (unparseTranslator.isEnabled()) {
         for (ASTNode sampleCol : sampleCols) {
           unparseTranslator.addIdentifierTranslation((ASTNode) sampleCol
@@ -1631,7 +1634,7 @@ public class SemanticAnalyzer extends Ba
 
         if (qb.getParseInfo().isAnalyzeCommand()) {
           // allow partial partition specification for nonscan since noscan is fast.
-          tableSpec ts = new tableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan);
+          TableSpec ts = new TableSpec(db, conf, (ASTNode) ast.getChild(0), true, this.noscan);
           if (ts.specType == SpecType.DYNAMIC_PARTITION) { // dynamic partitions
             try {
               ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec);
@@ -1663,6 +1666,7 @@ public class SemanticAnalyzer extends Ba
             }
           }
 
+          tab.setTableSpec(ts);
           qb.getParseInfo().addTableSpec(alias, ts);
         }
 
@@ -1704,7 +1708,7 @@ public class SemanticAnalyzer extends Ba
         ASTNode ast = qbp.getDestForClause(name);
         switch (ast.getToken().getType()) {
         case HiveParser.TOK_TAB: {
-          tableSpec ts = new tableSpec(db, conf, ast);
+          TableSpec ts = new TableSpec(db, conf, ast);
           if (ts.tableHandle.isView()) {
             throw new SemanticException(ErrorMsg.DML_AGAINST_VIEW.getMsg());
           }
@@ -1716,7 +1720,7 @@ public class SemanticAnalyzer extends Ba
                 .getMsg(ast, "The class is " + outputFormatClass.toString()));
           }
 
-          // tableSpec ts is got from the query (user specified),
+          // TableSpec ts is got from the query (user specified),
           // which means the user didn't specify partitions in their query,
           // but whether the table itself is partitioned is not know.
           if (ts.specType != SpecType.STATIC_PARTITION) {
@@ -1769,7 +1773,7 @@ public class SemanticAnalyzer extends Ba
                     "Error creating temporary folder on: " + location.toString()), e);
               }
               if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
-                tableSpec ts = new tableSpec(db, conf, this.ast);
+                TableSpec ts = new TableSpec(db, conf, this.ast);
                 // Set that variable to automatically collect stats during the MapReduce job
                 qb.getParseInfo().setIsInsertToTable(true);
                 // Add the table spec for the destination table.
@@ -9328,9 +9332,11 @@ public class SemanticAnalyzer extends Ba
     }
 
     // check if this table is sampled and needs more than input pruning
-    Operator<? extends OperatorDesc> tableOp = top;
+    Operator<? extends OperatorDesc> op = top;
     TableSample ts = qb.getParseInfo().getTabSample(alias);
     if (ts != null) {
+      TableScanOperator tableScanOp = (TableScanOperator) top;
+      tableScanOp.getConf().setTableSample(ts);
       int num = ts.getNumerator();
       int den = ts.getDenominator();
       ArrayList<ASTNode> sampleExprs = ts.getExprs();
@@ -9389,7 +9395,7 @@ public class SemanticAnalyzer extends Ba
         LOG.info("No need for sample filter");
         ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols,
             colsEqual, alias, rwsch, qb.getMetaData(), null);
-        tableOp = OperatorFactory.getAndMakeChild(new FilterDesc(
+        op = OperatorFactory.getAndMakeChild(new FilterDesc(
             samplePredicate, true, new SampleDesc(ts.getNumerator(), ts
                 .getDenominator(), tabBucketCols, true)),
             new RowSchema(rwsch.getColumnInfos()), top);
@@ -9399,7 +9405,7 @@ public class SemanticAnalyzer extends Ba
         LOG.info("Need sample filter");
         ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols,
             colsEqual, alias, rwsch, qb.getMetaData(), null);
-        tableOp = OperatorFactory.getAndMakeChild(new FilterDesc(
+        op = OperatorFactory.getAndMakeChild(new FilterDesc(
             samplePredicate, true),
             new RowSchema(rwsch.getColumnInfos()), top);
       }
@@ -9429,7 +9435,7 @@ public class SemanticAnalyzer extends Ba
             qb.getParseInfo().setTabSample(alias, tsSample);
             ExprNodeDesc samplePred = genSamplePredicate(tsSample, tab
                 .getBucketCols(), true, alias, rwsch, qb.getMetaData(), null);
-            tableOp = OperatorFactory
+            op = OperatorFactory
                 .getAndMakeChild(new FilterDesc(samplePred, true,
                     new SampleDesc(tsSample.getNumerator(), tsSample
                         .getDenominator(), tab.getBucketCols(), true)),
@@ -9447,7 +9453,7 @@ public class SemanticAnalyzer extends Ba
                     .valueOf(460476415)));
             ExprNodeDesc samplePred = genSamplePredicate(tsSample, null, false,
                 alias, rwsch, qb.getMetaData(), randFunc);
-            tableOp = OperatorFactory.getAndMakeChild(new FilterDesc(
+            op = OperatorFactory.getAndMakeChild(new FilterDesc(
                 samplePred, true),
                 new RowSchema(rwsch.getColumnInfos()), top);
           }
@@ -9455,10 +9461,10 @@ public class SemanticAnalyzer extends Ba
       }
     }
 
-    Operator output = putOpInsertMap(tableOp, rwsch);
+    Operator output = putOpInsertMap(op, rwsch);
 
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Created Table Plan for " + alias + " " + tableOp.toString());
+      LOG.debug("Created Table Plan for " + alias + " " + op.toString());
     }
 
     return output;
@@ -9503,7 +9509,7 @@ public class SemanticAnalyzer extends Ba
       tsDesc.addVirtualCols(vcList);
 
       String tblName = tab.getTableName();
-      tableSpec tblSpec = qbp.getTableSpec(alias);
+      TableSpec tblSpec = qbp.getTableSpec(alias);
       Map<String, String> partSpec = tblSpec.getPartSpec();
 
       if (partSpec != null) {
@@ -10014,14 +10020,15 @@ public class SemanticAnalyzer extends Ba
     }
 
     // 4. Generate Parse Context for Optimizer & Physical compiler
-    ParseContext pCtx = new ParseContext(conf, qb,
-        opToPartPruner, opToPartList, topOps,
+    copyInfoToQueryProperties(queryProperties);
+    ParseContext pCtx = new ParseContext(conf, opToPartPruner, opToPartList, topOps,
         new HashSet<JoinOperator>(joinContext.keySet()),
         new HashSet<SMBMapJoinOperator>(smbMapJoinContext.keySet()),
         loadTableWork, loadFileWork, ctx, idToTableNameMap, destTableId, uCtx,
         listMapJoinOpsNoReducer, prunedPartitions, opToSamplePruner,
         globalLimitCtx, nameToSplitSample, inputs, rootTasks, opToPartToSkewedPruner,
-        viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting, queryProperties);
+        viewAliasToInput, reduceSinkOperatorsAddedByEnforceBucketingSorting,
+        analyzeRewrite, tableDesc, queryProperties);
 
     // 5. Take care of view creation
     if (createVwDesc != null) {
@@ -10792,13 +10799,12 @@ public class SemanticAnalyzer extends Ba
     }
 
     // Handle different types of CREATE TABLE command
-    CreateTableDesc crtTblDesc = null;
     switch (command_type) {
 
     case CREATE_TABLE: // REGULAR CREATE TABLE DDL
       tblProps = addDefaultProperties(tblProps);
 
-      crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols,
+      CreateTableDesc crtTblDesc = new CreateTableDesc(dbDotTab, isExt, isTemporary, cols, partCols,
           bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
           rowFormatParams.fieldEscape,
           rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim, rowFormatParams.lineDelim,
@@ -10850,16 +10856,16 @@ public class SemanticAnalyzer extends Ba
 
       tblProps = addDefaultProperties(tblProps);
 
-      crtTblDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
+      tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
           partCols, bucketCols, sortCols, numBuckets, rowFormatParams.fieldDelim,
           rowFormatParams.fieldEscape, rowFormatParams.collItemDelim, rowFormatParams.mapKeyDelim,
           rowFormatParams.lineDelim, comment, storageFormat.getInputFormat(),
           storageFormat.getOutputFormat(), location, storageFormat.getSerde(),
           storageFormat.getStorageHandler(), storageFormat.getSerdeProps(), tblProps, ifNotExists,
           skewedColNames, skewedValues);
-      crtTblDesc.setStoredAsSubDirectories(storedAsDirs);
-      crtTblDesc.setNullFormat(rowFormatParams.nullFormat);
-      qb.setTableDesc(crtTblDesc);
+      tableDesc.setStoredAsSubDirectories(storedAsDirs);
+      tableDesc.setNullFormat(rowFormatParams.nullFormat);
+      qb.setTableDesc(tableDesc);
 
       SessionState.get().setCommandType(HiveOperation.CREATETABLE_AS_SELECT);
 
@@ -12075,4 +12081,19 @@ public class SemanticAnalyzer extends Ba
     selexpr.addChild(dot);
     return selexpr;
   }
+
+  private void copyInfoToQueryProperties(QueryProperties queryProperties) {
+    if (qb != null) {
+      queryProperties.setQuery(qb.getIsQuery());
+      queryProperties.setAnalyzeCommand(qb.getParseInfo().isAnalyzeCommand());
+      queryProperties.setPartialScanAnalyzeCommand(qb.getParseInfo().isPartialScanAnalyzeCommand());
+      queryProperties.setNoScanAnalyzeCommand(qb.getParseInfo().isNoScanAnalyzeCommand());
+      queryProperties.setAnalyzeRewrite(qb.isAnalyzeRewrite());
+      queryProperties.setCTAS(qb.getTableDesc() != null);
+      queryProperties.setInsertToTable(qb.getParseInfo().isInsertToTable());
+      queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() &&
+              !qb.getParseInfo().getDestToOrderBy().isEmpty());
+      queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit());
+    }
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java Tue Mar 10 04:20:45 2015
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.hooks.W
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.AnalyzeRewriteContext;
 import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc;
 import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
 import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
@@ -86,13 +87,13 @@ public abstract class TaskCompiler {
 
     Context ctx = pCtx.getContext();
     GlobalLimitCtx globalLimitCtx = pCtx.getGlobalLimitCtx();
-    QB qb = pCtx.getQB();
     List<Task<MoveWork>> mvTask = new ArrayList<Task<MoveWork>>();
 
     List<LoadTableDesc> loadTableWork = pCtx.getLoadTableWork();
     List<LoadFileDesc> loadFileWork = pCtx.getLoadFileWork();
 
-    boolean isCStats = qb.isAnalyzeRewrite();
+    boolean isCStats = pCtx.getQueryProperties().isAnalyzeRewrite();
+    int outerQueryLimit = pCtx.getQueryProperties().getOuterQueryLimit();
 
     if (pCtx.getFetchTask() != null) {
       return;
@@ -105,7 +106,7 @@ public abstract class TaskCompiler {
      * If the select is from analyze table column rewrite, don't create a fetch task. Instead create
      * a column stats task later.
      */
-    if (pCtx.getQB().getIsQuery() && !isCStats) {
+    if (pCtx.getQueryProperties().isQuery() && !isCStats) {
       if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
         throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
       }
@@ -115,14 +116,13 @@ public abstract class TaskCompiler {
       String cols = loadFileDesc.getColumns();
       String colTypes = loadFileDesc.getColumnTypes();
 
-      TableDesc resultTab = pCtx.getFetchTabledesc();
+      TableDesc resultTab = pCtx.getFetchTableDesc();
       if (resultTab == null) {
         String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
         resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
       }
 
-      FetchWork fetch = new FetchWork(loadFileDesc.getSourcePath(),
-                                      resultTab, qb.getParseInfo().getOuterQueryLimit());
+      FetchWork fetch = new FetchWork(loadFileDesc.getSourcePath(), resultTab, outerQueryLimit);
       fetch.setSource(pCtx.getFetchSource());
       fetch.setSink(pCtx.getFetchSink());
 
@@ -138,7 +138,7 @@ public abstract class TaskCompiler {
         globalLimitCtx.disableOpt();
 
       }
-      if (qb.getParseInfo().getOuterQueryLimit() == 0) {
+      if (outerQueryLimit == 0) {
         // Believe it or not, some tools do generate queries with limit 0 and than expect
         // query to run quickly. Lets meet their requirement.
         LOG.info("Limit 0. No query execution needed.");
@@ -166,17 +166,18 @@ public abstract class TaskCompiler {
 
       boolean oneLoadFile = true;
       for (LoadFileDesc lfd : loadFileWork) {
-        if (qb.isCTAS()) {
+        if (pCtx.getQueryProperties().isCTAS()) {
           assert (oneLoadFile); // should not have more than 1 load file for
           // CTAS
           // make the movetask's destination directory the table's destination.
           Path location;
-          String loc = qb.getTableDesc().getLocation();
+          String loc = pCtx.getCreateTable().getLocation();
           if (loc == null) {
             // get the table's default location
             Path targetPath;
             try {
-              String[] names = Utilities.getDbTableName(qb.getTableDesc().getTableName());
+              String[] names = Utilities.getDbTableName(
+                      pCtx.getCreateTable().getTableName());
               if (!db.databaseExists(names[0])) {
                 throw new SemanticException("ERROR: The database " + names[0]
                     + " does not exist.");
@@ -208,7 +209,8 @@ public abstract class TaskCompiler {
      * a column stats task instead of a fetch task to persist stats to the metastore.
      */
     if (isCStats) {
-      genColumnStatsTask(qb, loadTableWork, loadFileWork, rootTasks);
+      genColumnStatsTask(pCtx.getAnalyzeRewrite(), loadTableWork, loadFileWork,
+            rootTasks, outerQueryLimit);
     }
 
     // For each task, set the key descriptor for the reducer
@@ -226,9 +228,9 @@ public abstract class TaskCompiler {
 
     decideExecMode(rootTasks, ctx, globalLimitCtx);
 
-    if (qb.isCTAS()) {
+    if (pCtx.getQueryProperties().isCTAS()) {
       // generate a DDL task and make it a dependent task of the leaf
-      CreateTableDesc crtTblDesc = qb.getTableDesc();
+      CreateTableDesc crtTblDesc = pCtx.getCreateTable();
 
       crtTblDesc.validate(conf);
 
@@ -305,16 +307,15 @@ public abstract class TaskCompiler {
    * @param qb
    */
   @SuppressWarnings("unchecked")
-  protected void genColumnStatsTask(QB qb, List<LoadTableDesc> loadTableWork,
-      List<LoadFileDesc> loadFileWork, List<Task<? extends Serializable>> rootTasks) {
-    QBParseInfo qbParseInfo = qb.getParseInfo();
+  protected void genColumnStatsTask(AnalyzeRewriteContext analyzeRewrite, List<LoadTableDesc> loadTableWork,
+      List<LoadFileDesc> loadFileWork, List<Task<? extends Serializable>> rootTasks, int outerQueryLimit) {
     ColumnStatsTask cStatsTask = null;
     ColumnStatsWork cStatsWork = null;
     FetchWork fetch = null;
-    String tableName = qbParseInfo.getTableName();
-    List<String> colName = qbParseInfo.getColName();
-    List<String> colType = qbParseInfo.getColType();
-    boolean isTblLevel = qbParseInfo.isTblLvl();
+    String tableName = analyzeRewrite.getTableName();
+    List<String> colName = analyzeRewrite.getColName();
+    List<String> colType = analyzeRewrite.getColType();
+    boolean isTblLevel = analyzeRewrite.isTblLvl();
 
     String cols = loadFileWork.get(0).getColumns();
     String colTypes = loadFileWork.get(0).getColumnTypes();
@@ -322,8 +323,7 @@ public abstract class TaskCompiler {
     String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
     TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
 
-    fetch = new FetchWork(loadFileWork.get(0).getSourcePath(),
-        resultTab, qb.getParseInfo().getOuterQueryLimit());
+    fetch = new FetchWork(loadFileWork.get(0).getSourcePath(), resultTab, outerQueryLimit);
 
     ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName,
         colName, colType, isTblLevel);
@@ -390,7 +390,6 @@ public abstract class TaskCompiler {
    */
   public ParseContext getParseContext(ParseContext pCtx, List<Task<? extends Serializable>> rootTasks) {
     ParseContext clone = new ParseContext(conf,
-        pCtx.getQB(),
         pCtx.getOpToPartPruner(), pCtx.getOpToPartList(), pCtx.getTopOps(),
         pCtx.getJoinOps(), pCtx.getSmbMapJoinOps(),
         pCtx.getLoadTableWork(), pCtx.getLoadFileWork(), pCtx.getContext(),
@@ -400,7 +399,7 @@ public abstract class TaskCompiler {
         pCtx.getNameToSplitSample(), pCtx.getSemanticInputs(), rootTasks,
         pCtx.getOpToPartToSkewedPruner(), pCtx.getViewAliasToInput(),
         pCtx.getReduceSinkOperatorsAddedByEnforceBucketingSorting(),
-        pCtx.getQueryProperties());
+        pCtx.getAnalyzeRewrite(), pCtx.getCreateTable(), pCtx.getQueryProperties());
     clone.setFetchTask(pCtx.getFetchTask());
     clone.setLineageInfo(pCtx.getLineageInfo());
     clone.setMapJoinOps(pCtx.getMapJoinOps());

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Mar 10 04:20:45 2015
@@ -555,7 +555,7 @@ public class TypeCheckProcFactory {
 
   private static ExprNodeDesc toExprNodeDesc(ColumnInfo colInfo) {
     ObjectInspector inspector = colInfo.getObjectInspector();
-    if (inspector instanceof ConstantObjectInspector && 
+    if (inspector instanceof ConstantObjectInspector &&
         inspector instanceof PrimitiveObjectInspector) {
       PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
       Object constant = ((ConstantObjectInspector) inspector).getWritableConstantValue();
@@ -994,8 +994,15 @@ public class TypeCheckProcFactory {
           .getText());
       // NOTE: tableAlias must be a valid non-ambiguous table alias,
       // because we've checked that in TOK_TABLE_OR_COL's process method.
-      ColumnInfo colInfo = input.get(tableAlias, ((ExprNodeConstantDesc) nodeOutputs[1]).getValue()
-          .toString());
+      String colName;
+      if (nodeOutputs[1] instanceof ExprNodeConstantDesc) {
+        colName = ((ExprNodeConstantDesc) nodeOutputs[1]).getValue().toString();
+      }  else if (nodeOutputs[1] instanceof ExprNodeColumnDesc) {
+        colName = ((ExprNodeColumnDesc)nodeOutputs[1]).getColumn();
+      } else {
+        throw new SemanticException("Unexpected ExprNode : " + nodeOutputs[1]);
+      }
+      ColumnInfo colInfo = input.get(tableAlias, colName);
 
       if (colInfo == null) {
         ctx.setError(ErrorMsg.INVALID_COLUMN.getMsg(expr.getChild(1)), expr);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkProcessAnalyzeTable.java Tue Mar 10 04:20:45 2015
@@ -40,8 +40,6 @@ import org.apache.hadoop.hive.ql.metadat
 import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
 import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
-import org.apache.hadoop.hive.ql.parse.QB;
-import org.apache.hadoop.hive.ql.parse.QBParseInfo;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.SparkWork;
@@ -84,10 +82,8 @@ public class SparkProcessAnalyzeTable im
     @SuppressWarnings("rawtypes")
     Class<? extends InputFormat> inputFormat = tableScan.getConf().getTableMetadata()
         .getInputFormatClass();
-    QB queryBlock = parseContext.getQB();
-    QBParseInfo parseInfo = parseContext.getQB().getParseInfo();
 
-    if (parseInfo.isAnalyzeCommand()) {
+    if (parseContext.getQueryProperties().isAnalyzeCommand()) {
       Preconditions.checkArgument(tableScan.getChildOperators() == null
         || tableScan.getChildOperators().size() == 0,
         "AssertionError: expected tableScan.getChildOperators() to be null, "
@@ -102,14 +98,14 @@ public class SparkProcessAnalyzeTable im
       Preconditions.checkArgument(alias != null, "AssertionError: expected alias to be not null");
 
       SparkWork sparkWork = context.currentTask.getWork();
-      boolean partialScan = parseInfo.isPartialScanAnalyzeCommand();
-      boolean noScan = parseInfo.isNoScanAnalyzeCommand();
+      boolean partialScan = parseContext.getQueryProperties().isPartialScanAnalyzeCommand();
+      boolean noScan = parseContext.getQueryProperties().isNoScanAnalyzeCommand();
       if (inputFormat.equals(OrcInputFormat.class) && (noScan || partialScan)) {
 
         // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
         // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
         // There will not be any Spark job above this task
-        StatsNoJobWork snjWork = new StatsNoJobWork(parseContext.getQB().getParseInfo().getTableSpec());
+        StatsNoJobWork snjWork = new StatsNoJobWork(tableScan.getConf().getTableMetadata().getTableSpec());
         snjWork.setStatsReliable(parseContext.getConf().getBoolVar(
             HiveConf.ConfVars.HIVE_STATS_RELIABLE));
         Task<StatsNoJobWork> snjTask = TaskFactory.get(snjWork, parseContext.getConf());
@@ -123,7 +119,7 @@ public class SparkProcessAnalyzeTable im
         // The plan consists of a simple SparkTask followed by a StatsTask.
         // The Spark task is just a simple TableScanOperator
 
-        StatsWork statsWork = new StatsWork(parseInfo.getTableSpec());
+        StatsWork statsWork = new StatsWork(tableScan.getConf().getTableMetadata().getTableSpec());
         statsWork.setAggKey(tableScan.getConf().getStatsAggPrefix());
         statsWork.setSourceTask(context.currentTask);
         statsWork.setStatsReliable(parseContext.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STATS_RELIABLE));
@@ -132,7 +128,7 @@ public class SparkProcessAnalyzeTable im
 
         // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
         // The plan consists of a StatsTask only.
-        if (parseInfo.isNoScanAnalyzeCommand()) {
+        if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) {
           statsTask.setParentTasks(null);
           statsWork.setNoScanAnalyzeCommand(true);
           context.rootTasks.remove(context.currentTask);
@@ -140,17 +136,17 @@ public class SparkProcessAnalyzeTable im
         }
 
         // ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS partialscan;
-        if (parseInfo.isPartialScanAnalyzeCommand()) {
-          handlePartialScanCommand(tableScan, parseContext, parseInfo, statsWork, context, statsTask);
+        if (parseContext.getQueryProperties().isPartialScanAnalyzeCommand()) {
+          handlePartialScanCommand(tableScan, parseContext, statsWork, context, statsTask);
         }
 
         // NOTE: here we should use the new partition predicate pushdown API to get a list of pruned list,
         // and pass it to setTaskPlan as the last parameter
-        Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(parseInfo);
+        Set<Partition> confirmedPartns = GenMapRedUtils.getConfirmedPartitionsForScan(tableScan);
         PrunedPartitionList partitions = null;
         if (confirmedPartns.size() > 0) {
-          Table source = queryBlock.getMetaData().getTableForAlias(alias);
-          List<String> partCols = GenMapRedUtils.getPartitionColumns(parseInfo);
+          Table source = tableScan.getConf().getTableMetadata();
+          List<String> partCols = GenMapRedUtils.getPartitionColumns(tableScan);
           partitions = new PrunedPartitionList(source, confirmedPartns, partCols, false);
         }
 
@@ -169,11 +165,11 @@ public class SparkProcessAnalyzeTable im
    * It is composed of PartialScanTask followed by StatsTask.
    */
   private void handlePartialScanCommand(TableScanOperator tableScan, ParseContext parseContext,
-      QBParseInfo parseInfo, StatsWork statsWork, GenSparkProcContext context,
-      Task<StatsWork> statsTask) throws SemanticException {
+      StatsWork statsWork, GenSparkProcContext context, Task<StatsWork> statsTask)
+              throws SemanticException {
     String aggregationKey = tableScan.getConf().getStatsAggPrefix();
     StringBuffer aggregationKeyBuffer = new StringBuffer(aggregationKey);
-    List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(parseInfo, aggregationKeyBuffer);
+    List<Path> inputPaths = GenMapRedUtils.getInputPathsForPartialScan(tableScan, aggregationKeyBuffer);
     aggregationKey = aggregationKeyBuffer.toString();
 
     // scan work

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/DropTableDesc.java Tue Mar 10 04:20:45 2015
@@ -72,7 +72,7 @@ public class DropTableDesc extends DDLDe
   }
 
   public DropTableDesc(String tableName, Map<Integer, List<ExprNodeGenericFuncDesc>> partSpecs,
-      boolean expectView, boolean ignoreProtection) {
+      boolean expectView, boolean ignoreProtection, boolean ifPurge) {
     this.tableName = tableName;
     this.partSpecs = new ArrayList<PartSpec>(partSpecs.size());
     for (Map.Entry<Integer, List<ExprNodeGenericFuncDesc>> partSpec : partSpecs.entrySet()) {
@@ -83,6 +83,7 @@ public class DropTableDesc extends DDLDe
     }
     this.ignoreProtection = ignoreProtection;
     this.expectView = expectView;
+    this.ifPurge = ifPurge;
   }
 
   /**

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Tue Mar 10 04:20:45 2015
@@ -32,6 +32,10 @@ import org.apache.hadoop.hive.ql.udf.gen
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.apache.hadoop.util.ReflectionUtils;
 
@@ -427,4 +431,17 @@ public class ExprNodeDescUtils {
     }
     return true;
   }
+
+  public static PrimitiveTypeInfo deriveMinArgumentCast(
+      ExprNodeDesc childExpr, TypeInfo targetType) {
+    assert targetType instanceof PrimitiveTypeInfo : "Not a primitive type" + targetType;
+    PrimitiveTypeInfo pti = (PrimitiveTypeInfo)targetType;
+    // We only do the minimum cast for decimals. Other types are assumed safe; fix if needed.
+    // We also don't do anything for non-primitive children (maybe we should assert).
+    if ((pti.getPrimitiveCategory() != PrimitiveCategory.DECIMAL)
+        || (!(childExpr.getTypeInfo() instanceof PrimitiveTypeInfo))) return pti;
+    PrimitiveTypeInfo childTi = (PrimitiveTypeInfo)childExpr.getTypeInfo();
+    // If the child is also decimal, no cast is needed (we hope - can target type be narrower?).
+    return HiveDecimalUtils.getDecimalTypeForPrimitiveCategory(childTi);
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java Tue Mar 10 04:20:45 2015
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hive.ql.plan;
 
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -80,7 +79,7 @@ public class MergeJoinWork extends BaseW
     }
   }
 
-  @Explain(skipHeader=true, displayName = "Join")
+  @Explain(skipHeader = true, displayName = "Join")
   public List<BaseWork> getBaseWorkList() {
     return mergeWorkList;
   }
@@ -89,7 +88,7 @@ public class MergeJoinWork extends BaseW
     return ((MapWork) bigTableWork).getAliasToWork().keySet().iterator().next();
   }
 
-  @Explain(skipHeader=true, displayName = "Main")
+  @Explain(skipHeader = true, displayName = "Main")
   public BaseWork getMainWork() {
     return bigTableWork;
   }
@@ -103,4 +102,9 @@ public class MergeJoinWork extends BaseW
   public void addDummyOp(HashTableDummyOperator dummyOp) {
     getMainWork().addDummyOp(dummyOp);
   }
+
+  @Override
+  public List<HashTableDummyOperator> getDummyOps() {
+    return getMainWork().getDummyOps();
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PTFDeserializer.java Tue Mar 10 04:20:45 2015
@@ -27,7 +27,7 @@ import java.util.Map;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
 import org.apache.hadoop.hive.ql.exec.PTFPartition;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -288,7 +288,7 @@ public class PTFDeserializer {
     try {
       @SuppressWarnings("unchecked")
       Class<? extends TableFunctionResolver> rCls = (Class<? extends TableFunctionResolver>)
-          Class.forName(className);
+          JavaUtils.loadClass(className);
       return ReflectionUtils.newInstance(rCls, null);
     } catch (Exception e) {
       throw new HiveException(e);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Tue Mar 10 04:20:45 2015
@@ -31,6 +31,7 @@ import java.util.Set;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -144,7 +145,7 @@ public final class PlanUtils {
             serdeConstants.SERIALIZATION_LIB, localDirectoryDesc.getSerName());
       }
       if (localDirectoryDesc.getOutputFormat() != null){
-          ret.setOutputFileFormatClass(Class.forName(localDirectoryDesc.getOutputFormat()));
+        ret.setOutputFileFormatClass(JavaUtils.loadClass(localDirectoryDesc.getOutputFormat()));
       }
       if (localDirectoryDesc.getNullFormat() != null) {
         properties.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT,
@@ -307,7 +308,7 @@ public final class PlanUtils {
 
     try {
       if (crtTblDesc.getSerName() != null) {
-        Class c = Class.forName(crtTblDesc.getSerName());
+        Class c = JavaUtils.loadClass(crtTblDesc.getSerName());
         serdeClass = c;
       }
 
@@ -356,8 +357,8 @@ public final class PlanUtils {
 
       // replace the default input & output file format with those found in
       // crtTblDesc
-      Class c1 = Class.forName(crtTblDesc.getInputFormat());
-      Class c2 = Class.forName(crtTblDesc.getOutputFormat());
+      Class c1 = JavaUtils.loadClass(crtTblDesc.getInputFormat());
+      Class c2 = JavaUtils.loadClass(crtTblDesc.getOutputFormat());
       Class<? extends InputFormat> in_class = c1;
       Class<? extends HiveOutputFormat> out_class = c2;
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsNoJobWork.java Tue Mar 10 04:20:45 2015
@@ -20,9 +20,10 @@ package org.apache.hadoop.hive.ql.plan;
 
 import java.io.Serializable;
 
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
 import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
 
+
 /**
  * Client-side stats aggregator task.
  */
@@ -30,14 +31,14 @@ import org.apache.hadoop.hive.ql.parse.P
 public class StatsNoJobWork implements Serializable {
   private static final long serialVersionUID = 1L;
 
-  private tableSpec tableSpecs;
+  private TableSpec tableSpecs;
   private boolean statsReliable;
   private PrunedPartitionList prunedPartitionList;
 
   public StatsNoJobWork() {
   }
 
-  public StatsNoJobWork(tableSpec tableSpecs) {
+  public StatsNoJobWork(TableSpec tableSpecs) {
     this.tableSpecs = tableSpecs;
   }
 
@@ -45,7 +46,7 @@ public class StatsNoJobWork implements S
     this.statsReliable = statsReliable;
   }
 
-  public tableSpec getTableSpecs() {
+  public TableSpec getTableSpecs() {
     return tableSpecs;
   }
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/StatsWork.java Tue Mar 10 04:20:45 2015
@@ -21,7 +21,7 @@ package org.apache.hadoop.hive.ql.plan;
 import java.io.Serializable;
 
 import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec;
+import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
 
 /**
  * ConditionalStats.
@@ -31,7 +31,7 @@ import org.apache.hadoop.hive.ql.parse.B
 public class StatsWork implements Serializable {
   private static final long serialVersionUID = 1L;
 
-  private tableSpec tableSpecs;         // source table spec -- for TableScanOperator
+  private TableSpec tableSpecs;         // source table spec -- for TableScanOperator
   private LoadTableDesc loadTableDesc;  // same as MoveWork.loadTableDesc -- for FileSinkOperator
   private LoadFileDesc loadFileDesc;    // same as MoveWork.loadFileDesc -- for FileSinkOperator
   private String aggKey;                // aggregation key prefix
@@ -58,7 +58,7 @@ public class StatsWork implements Serial
   public StatsWork() {
   }
 
-  public StatsWork(tableSpec tableSpecs) {
+  public StatsWork(TableSpec tableSpecs) {
     this.tableSpecs = tableSpecs;
   }
 
@@ -74,7 +74,7 @@ public class StatsWork implements Serial
     this.statsReliable = statsReliable;
   }
 
-  public tableSpec getTableSpecs() {
+  public TableSpec getTableSpecs() {
     return tableSpecs;
   }
 

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java Tue Mar 10 04:20:45 2015
@@ -26,6 +26,7 @@ import java.util.Map;
 import org.apache.hadoop.hive.ql.exec.PTFUtils;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
+import org.apache.hadoop.hive.ql.parse.TableSample;
 
 /**
  * Table Scan Descriptor Currently, data is only read from a base source as part
@@ -91,9 +92,11 @@ public class TableScanDesc extends Abstr
 
   // input file name (big) to bucket number
   private Map<String, Integer> bucketFileNameMapping;
-  
+
   private boolean isMetadataOnly = false;
 
+  private transient TableSample tableSample;
+
   private transient final Table tableMetadata;
 
 
@@ -268,4 +271,12 @@ public class TableScanDesc extends Abstr
   public Table getTableMetadata() {
     return tableMetadata;
   }
+
+  public TableSample getTableSample() {
+    return tableSample;
+  }
+
+  public void setTableSample(TableSample tableSample) {
+    this.tableSample = tableSample;
+  }
 }

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/PTFInputDef.java Tue Mar 10 04:20:45 2015
@@ -19,9 +19,9 @@
 package org.apache.hadoop.hive.ql.plan.ptf;
 
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.hive.ql.exec.RowSchema;
 import org.apache.hadoop.hive.ql.plan.Explain;
-import org.apache.hadoop.util.StringUtils;
 
 public abstract class PTFInputDef {
   private String expressionTreeString;
@@ -43,7 +43,7 @@ public abstract class PTFInputDef {
   @Explain(displayName = "output shape")
   public String getOutputShapeExplain() {
     RowSchema schema = outputShape.getRr().getRowSchema();
-    return StringUtils.join(", ", schema.getSignature());
+    return StringUtils.join(schema.getSignature(), ", ");
   }
 
   public void setOutputShape(ShapeDetails outputShape) {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Tue Mar 10 04:20:45 2015
@@ -588,8 +588,15 @@ public class SessionState {
       Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true);
     }
     FsPermission currentHDFSDirPermission = fs.getFileStatus(rootHDFSDirPath).getPermission();
-    LOG.debug("HDFS root scratch dir: " + rootHDFSDirPath + ", permission: "
-        + currentHDFSDirPermission);
+    if (rootHDFSDirPath != null && rootHDFSDirPath.toUri() != null) {
+      String schema = rootHDFSDirPath.toUri().getScheme();
+      LOG.debug(
+        "HDFS root scratch dir: " + rootHDFSDirPath + " with schema " + schema + ", permission: " +
+          currentHDFSDirPermission);
+    } else {
+      LOG.debug(
+        "HDFS root scratch dir: " + rootHDFSDirPath + ", permission: " + currentHDFSDirPermission);
+    }
     // If the root HDFS scratch dir already exists, make sure it is writeable.
     if (!((currentHDFSDirPermission.toShort() & writableHDFSDirPermission
         .toShort()) == writableHDFSDirPermission.toShort())) {

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Tue Mar 10 04:20:45 2015
@@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Task;
 import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -69,7 +70,7 @@ public class JDBCStatsAggregator impleme
     this.sourceTask = sourceTask;
 
     try {
-      Class.forName(driver).newInstance();
+      JavaUtils.loadClass(driver).newInstance();
     } catch (Exception e) {
       LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
       return false;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Tue Mar 10 04:20:45 2015
@@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
@@ -69,7 +70,7 @@ public class JDBCStatsPublisher implemen
     String driver = HiveConf.getVar(hiveconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER);
 
     try {
-      Class.forName(driver).newInstance();
+      JavaUtils.loadClass(driver).newInstance();
     } catch (Exception e) {
       LOG.error("Error during instantiating JDBC driver " + driver + ". ", e);
       return false;
@@ -272,7 +273,7 @@ public class JDBCStatsPublisher implemen
       this.hiveconf = hconf;
       connectionString = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSDBCONNECTIONSTRING);
       String driver = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESTATSJDBCDRIVER);
-      Class.forName(driver).newInstance();
+      JavaUtils.loadClass(driver).newInstance();
       synchronized(DriverManager.class) {
         DriverManager.setLoginTimeout(timeout);
         conn = DriverManager.getConnection(connectionString);

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java Tue Mar 10 04:20:45 2015
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.BlockLocatio
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.common.ValidReadTxnList;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -647,7 +648,7 @@ public class CompactorMR {
   private static <T> T instantiate(Class<T> classType, String classname) throws IOException {
     T t = null;
     try {
-      Class c = Class.forName(classname);
+      Class c = JavaUtils.loadClass(classname);
       Object o = c.newInstance();
       if (classType.isAssignableFrom(o.getClass())) {
         t = (T)o;

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java?rev=1665403&r1=1665402&r2=1665403&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFnGrams.java Tue Mar 10 04:20:45 2015
@@ -231,6 +231,13 @@ public class GenericUDAFnGrams implement
       NGramAggBuf myagg = (NGramAggBuf) agg;
       List<Text> partialNGrams = (List<Text>) loi.getList(partial);
       int n = Integer.parseInt(partialNGrams.get(partialNGrams.size()-1).toString());
+
+      // A value of 0 for n indicates that the mapper processed data that does not meet
+      // filter criteria, so merge() should be NO-OP.
+      if (n == 0) {
+        return;
+      }
+
       if(myagg.n > 0 && myagg.n != n) {
         throw new HiveException(getClass().getSimpleName() + ": mismatch in value for 'n'"
             + ", which usually is caused by a non-constant expression. Found '"+n+"' and '"