You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ke...@apache.org on 2013/01/09 18:59:24 UTC

svn commit: r1430979 [2/3] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/test/quer...

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1430979&r1=1430978&r2=1430979&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Jan  9 17:59:23 2013
@@ -217,11 +217,11 @@ public class SemanticAnalyzer extends Ba
   private final UnparseTranslator unparseTranslator;
   private final GlobalLimitCtx globalLimitCtx = new GlobalLimitCtx();
 
-  //prefix for column names auto generated by hive
+  // prefix for column names auto generated by hive
   private final String autogenColAliasPrfxLbl;
   private final boolean autogenColAliasPrfxIncludeFuncName;
 
-  //Max characters when auto generating the column name with func name
+  // Max characters when auto generating the column name with func name
   private static final int AUTOGEN_COLALIAS_PRFX_MAXLENGTH = 20;
 
   private static class Phase1Ctx {
@@ -250,9 +250,9 @@ public class SemanticAnalyzer extends Ba
     prunedPartitions = new HashMap<String, PrunedPartitionList>();
     unparseTranslator = new UnparseTranslator();
     autogenColAliasPrfxLbl = HiveConf.getVar(conf,
-                                HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL);
+        HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_LABEL);
     autogenColAliasPrfxIncludeFuncName = HiveConf.getBoolVar(conf,
-                         HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME);
+        HiveConf.ConfVars.HIVE_AUTOGEN_COLUMNALIAS_PREFIX_INCLUDEFUNCNAME);
     queryProperties = new QueryProperties();
     opToPartToSkewedPruner = new HashMap<TableScanOperator, Map<String, ExprNodeDesc>>();
   }
@@ -355,7 +355,8 @@ public class SemanticAnalyzer extends Ba
       ASTNode selectExpr, QBParseInfo qbp) {
     for (int i = 0; i < selectExpr.getChildCount(); ++i) {
       ASTNode selExpr = (ASTNode) selectExpr.getChild(i);
-      if ((selExpr.getToken().getType() == HiveParser.TOK_SELEXPR) && (selExpr.getChildCount() == 2)) {
+      if ((selExpr.getToken().getType() == HiveParser.TOK_SELEXPR)
+          && (selExpr.getChildCount() == 2)) {
         String columnAlias = unescapeIdentifier(selExpr.getChild(1).getText());
         qbp.setExprToColumnAlias((ASTNode) selExpr.getChild(0), columnAlias);
       }
@@ -490,15 +491,15 @@ public class SemanticAnalyzer extends Ba
       // Need to change it to list of columns
       if (sampleCols.size() > 2) {
         throw new SemanticException(generateErrorMessage(
-              (ASTNode) tabref.getChild(0),
-              ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
+            (ASTNode) tabref.getChild(0),
+            ErrorMsg.SAMPLE_RESTRICTION.getMsg()));
       }
       qb.getParseInfo().setTabSample(
           alias,
           new TableSample(
-          unescapeIdentifier(sampleClause.getChild(0).getText()),
-          unescapeIdentifier(sampleClause.getChild(1).getText()),
-          sampleCols));
+              unescapeIdentifier(sampleClause.getChild(0).getText()),
+              unescapeIdentifier(sampleClause.getChild(1).getText()),
+              sampleCols));
       if (unparseTranslator.isEnabled()) {
         for (ASTNode sampleCol : sampleCols) {
           unparseTranslator.addIdentifierTranslation((ASTNode) sampleCol
@@ -622,7 +623,7 @@ public class SemanticAnalyzer extends Ba
     if ((numChildren != 2) && (numChildren != 3)
         && join.getToken().getType() != HiveParser.TOK_UNIQUEJOIN) {
       throw new SemanticException(generateErrorMessage(join,
-                  "Join with multiple children"));
+          "Join with multiple children"));
     }
 
     for (int num = 0; num < numChildren; num++) {
@@ -735,7 +736,7 @@ public class SemanticAnalyzer extends Ba
 
       case HiveParser.TOK_INSERT_INTO:
         String currentDatabase = db.getCurrentDatabase();
-        String tab_name = getUnescapedName((ASTNode)ast.getChild(0).getChild(0), currentDatabase);
+        String tab_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0), currentDatabase);
         qbp.addInsertIntoTable(tab_name);
 
       case HiveParser.TOK_DESTINATION:
@@ -759,7 +760,7 @@ public class SemanticAnalyzer extends Ba
         int child_count = ast.getChildCount();
         if (child_count != 1) {
           throw new SemanticException(generateErrorMessage(ast,
-                      "Multiple Children " + child_count));
+              "Multiple Children " + child_count));
         }
 
         // Check if this is a subquery / lateral view
@@ -792,10 +793,10 @@ public class SemanticAnalyzer extends Ba
         qbp.setDistributeByExprForClause(ctx_1.dest, ast);
         if (qbp.getClusterByForClause(ctx_1.dest) != null) {
           throw new SemanticException(generateErrorMessage(ast,
-                ErrorMsg.CLUSTERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
+              ErrorMsg.CLUSTERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
         } else if (qbp.getOrderByForClause(ctx_1.dest) != null) {
           throw new SemanticException(generateErrorMessage(ast,
-                ErrorMsg.ORDERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
+              ErrorMsg.ORDERBY_DISTRIBUTEBY_CONFLICT.getMsg()));
         }
         break;
 
@@ -806,10 +807,10 @@ public class SemanticAnalyzer extends Ba
         qbp.setSortByExprForClause(ctx_1.dest, ast);
         if (qbp.getClusterByForClause(ctx_1.dest) != null) {
           throw new SemanticException(generateErrorMessage(ast,
-                ErrorMsg.CLUSTERBY_SORTBY_CONFLICT.getMsg()));
+              ErrorMsg.CLUSTERBY_SORTBY_CONFLICT.getMsg()));
         } else if (qbp.getOrderByForClause(ctx_1.dest) != null) {
           throw new SemanticException(generateErrorMessage(ast,
-                ErrorMsg.ORDERBY_SORTBY_CONFLICT.getMsg()));
+              ErrorMsg.ORDERBY_SORTBY_CONFLICT.getMsg()));
         }
 
         break;
@@ -821,7 +822,7 @@ public class SemanticAnalyzer extends Ba
         qbp.setOrderByExprForClause(ctx_1.dest, ast);
         if (qbp.getClusterByForClause(ctx_1.dest) != null) {
           throw new SemanticException(generateErrorMessage(ast,
-                ErrorMsg.CLUSTERBY_ORDERBY_CONFLICT.getMsg()));
+              ErrorMsg.CLUSTERBY_ORDERBY_CONFLICT.getMsg()));
         }
         break;
 
@@ -837,7 +838,7 @@ public class SemanticAnalyzer extends Ba
         }
         if (qbp.getSelForClause(ctx_1.dest).getToken().getType() == HiveParser.TOK_SELECTDI) {
           throw new SemanticException(generateErrorMessage(ast,
-                ErrorMsg.SELECT_DISTINCT_WITH_GROUPBY.getMsg()));
+              ErrorMsg.SELECT_DISTINCT_WITH_GROUPBY.getMsg()));
         }
         qbp.setGroupByExprForClause(ctx_1.dest, ast);
         skipRecursion = true;
@@ -864,7 +865,7 @@ public class SemanticAnalyzer extends Ba
       case HiveParser.TOK_ANALYZE:
         // Case of analyze command
 
-        String table_name = getUnescapedName((ASTNode)ast.getChild(0).getChild(0));
+        String table_name = getUnescapedName((ASTNode) ast.getChild(0).getChild(0));
 
 
         qb.setTabAlias(table_name, table_name);
@@ -882,7 +883,7 @@ public class SemanticAnalyzer extends Ba
         // select * from (subq1 union subq2) subqalias
         if (!qbp.getIsSubQ()) {
           throw new SemanticException(generateErrorMessage(ast,
-                                        ErrorMsg.UNION_NOTIN_SUBQ.getMsg()));
+              ErrorMsg.UNION_NOTIN_SUBQ.getMsg()));
         }
 
       case HiveParser.TOK_INSERT:
@@ -925,7 +926,7 @@ public class SemanticAnalyzer extends Ba
             }
           } else {
             throw new SemanticException(ErrorMsg.INSERT_INTO_DYNAMICPARTITION_IFNOTEXISTS
-                  .getMsg(partition.toString()));
+                .getMsg(partition.toString()));
           }
         }
 
@@ -977,7 +978,7 @@ public class SemanticAnalyzer extends Ba
         }
 
         // Disallow INSERT INTO on bucketized tables
-        if(qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()) &&
+        if (qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()) &&
             tab.getNumBuckets() > 0) {
           throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE.
               getMsg("Table: " + tab_name));
@@ -999,9 +1000,9 @@ public class SemanticAnalyzer extends Ba
           if (qb.getParseInfo().isAnalyzeCommand()) {
             throw new SemanticException(ErrorMsg.ANALYZE_VIEW.getMsg());
           }
-          String fullViewName = tab.getDbName()+"."+tab.getTableName();
+          String fullViewName = tab.getDbName() + "." + tab.getTableName();
           // Prevent view cycles
-          if(viewsExpanded.contains(fullViewName)){
+          if (viewsExpanded.contains(fullViewName)) {
             throw new SemanticException("Recursive view " + fullViewName +
                 " detected (cycle: " + StringUtils.join(viewsExpanded, " -> ") +
                 " -> " + fullViewName + ").");
@@ -1016,8 +1017,8 @@ public class SemanticAnalyzer extends Ba
 
         if (!InputFormat.class.isAssignableFrom(tab.getInputFormatClass())) {
           throw new SemanticException(generateErrorMessage(
-                qb.getParseInfo().getSrcForAlias(alias),
-                ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg()));
+              qb.getParseInfo().getSrcForAlias(alias),
+              ErrorMsg.INVALID_INPUT_FORMAT_TYPE.getMsg()));
         }
 
         qb.getMetaData().setSrcForAlias(alias, tab);
@@ -1028,8 +1029,9 @@ public class SemanticAnalyzer extends Ba
             try {
               ts.partitions = db.getPartitionsByNames(ts.tableHandle, ts.partSpec);
             } catch (HiveException e) {
-              throw new SemanticException(generateErrorMessage(qb.getParseInfo().getSrcForAlias(alias),
-                          "Cannot get partitions for " + ts.partSpec), e);
+              throw new SemanticException(generateErrorMessage(
+                  qb.getParseInfo().getSrcForAlias(alias),
+                  "Cannot get partitions for " + ts.partSpec), e);
             }
           }
           qb.getParseInfo().addTableSpec(alias, ts);
@@ -1046,7 +1048,7 @@ public class SemanticAnalyzer extends Ba
         QBExpr qbexpr = qb.getSubqForAlias(alias);
         getMetaData(qbexpr);
         if (wasView) {
-           viewsExpanded.remove(viewsExpanded.size()-1);
+          viewsExpanded.remove(viewsExpanded.size() - 1);
         }
       }
 
@@ -1106,7 +1108,7 @@ public class SemanticAnalyzer extends Ba
               ctx.setResFile(null);
 
               // allocate a temporary output dir on the location of the table
-              String tableName = getUnescapedName((ASTNode)ast.getChild(0));
+              String tableName = getUnescapedName((ASTNode) ast.getChild(0));
               Table newTable = db.newTable(tableName);
               Path location;
               try {
@@ -1120,7 +1122,7 @@ public class SemanticAnalyzer extends Ba
                     FileUtils.makeQualified(location, conf).toUri());
               } catch (Exception e) {
                 throw new SemanticException(generateErrorMessage(ast,
-                      "Error creating temporary folder on: " + location.toString()), e);
+                    "Error creating temporary folder on: " + location.toString()), e);
               }
               if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESTATSAUTOGATHER)) {
                 tableSpec ts = new tableSpec(db, conf, this.ast);
@@ -1141,7 +1143,7 @@ public class SemanticAnalyzer extends Ba
         }
         default:
           throw new SemanticException(generateErrorMessage(ast,
-                "Unknown Token Type " + ast.getToken().getType()));
+              "Unknown Token Type " + ast.getToken().getType()));
         }
       }
     } catch (HiveException e) {
@@ -1159,7 +1161,7 @@ public class SemanticAnalyzer extends Ba
     ASTNode viewTree;
     final ASTNodeOrigin viewOrigin = new ASTNodeOrigin("VIEW", tab.getTableName(),
         tab.getViewExpandedText(), alias, qb.getParseInfo().getSrcForAlias(
-        alias));
+            alias));
     try {
       String viewText = tab.getViewExpandedText();
       // Reparse text, passing null for context to avoid clobbering
@@ -1341,6 +1343,7 @@ public class SemanticAnalyzer extends Ba
       joinTree.addFilterMapping(cond.getRight(), cond.getLeft(), filters.get(1).size());
     }
   }
+
   /**
    * Parse the join condition. If the condition is a join condition, throw an
    * error if it is not an equality. Otherwise, break it into left and right
@@ -1406,7 +1409,7 @@ public class SemanticAnalyzer extends Ba
         if ((rightCondAl1.size() != 0)
             || ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
           if (type.equals(JoinType.LEFTOUTER) ||
-                type.equals(JoinType.FULLOUTER)) {
+              type.equals(JoinType.FULLOUTER)) {
             if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
               joinTree.getFilters().get(0).add(joinCond);
             } else {
@@ -1493,7 +1496,7 @@ public class SemanticAnalyzer extends Ba
       for (int ci = childrenBegin; ci < joinCond.getChildCount(); ci++) {
         parseJoinCondPopulateAlias(joinTree, (ASTNode) joinCond.getChild(ci),
             leftAlias.get(ci - childrenBegin), rightAlias.get(ci
-            - childrenBegin), null);
+                - childrenBegin), null);
       }
 
       boolean leftAliasNull = true;
@@ -1574,7 +1577,7 @@ public class SemanticAnalyzer extends Ba
 
     Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
         new FilterDesc(genExprNodeDesc(condn, inputRR), false), new RowSchema(
-        inputRR.getColumnInfos()), input), inputRR);
+            inputRR.getColumnInfos()), input), inputRR);
 
     return output;
   }
@@ -1605,11 +1608,11 @@ public class SemanticAnalyzer extends Ba
     RowResolver inputRR = inputCtx.getRowResolver();
     Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
         new FilterDesc(genExprNodeDesc(condn, inputRR), false), new RowSchema(
-        inputRR.getColumnInfos()), input), inputRR);
+            inputRR.getColumnInfos()), input), inputRR);
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Created Filter Plan for " + qb.getId() + " row schema: "
-                + inputRR.toString());
+          + inputRR.toString());
     }
     return output;
   }
@@ -1677,8 +1680,8 @@ public class SemanticAnalyzer extends Ba
         col_list.add(expr);
         output.put(tmp[0], tmp[1],
             new ColumnInfo(getColumnInternalName(pos), colInfo.getType(),
-            colInfo.getTabAlias(), colInfo.getIsVirtualCol(),
-            colInfo.isHiddenVirtualCol()));
+                colInfo.getTabAlias(), colInfo.getIsVirtualCol(),
+                colInfo.isHiddenVirtualCol()));
         pos = Integer.valueOf(pos.intValue() + 1);
         matched++;
 
@@ -1800,7 +1803,7 @@ public class SemanticAnalyzer extends Ba
           tblDesc.getProperties().setProperty(serdeConstants.LINE_DELIM, lineDelim);
           if (!lineDelim.equals("\n") && !lineDelim.equals("10")) {
             throw new SemanticException(generateErrorMessage(rowChild,
-                    ErrorMsg.LINES_TERMINATED_BY_NON_NEWLINE.getMsg()));
+                ErrorMsg.LINES_TERMINATED_BY_NON_NEWLINE.getMsg()));
           }
           break;
         default:
@@ -1885,7 +1888,7 @@ public class SemanticAnalyzer extends Ba
           String intName = getColumnInternalName(i);
           ColumnInfo colInfo = new ColumnInfo(intName, TypeInfoUtils
               .getTypeInfoFromTypeString(getTypeStringFromAST((ASTNode) child
-              .getChild(1))), null, false);
+                  .getChild(1))), null, false);
           colInfo.setAlias(colAlias);
           outputCols.add(colInfo);
         }
@@ -1978,8 +1981,8 @@ public class SemanticAnalyzer extends Ba
 
     Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
         new ScriptDesc(
-        fetchFilesNotInLocalFilesystem(stripQuotes(trfm.getChild(execPos).getText())),
-        inInfo, inRecordWriter, outInfo, outRecordReader, errRecordReader, errInfo),
+            fetchFilesNotInLocalFilesystem(stripQuotes(trfm.getChild(execPos).getText())),
+            inInfo, inRecordWriter, outInfo, outRecordReader, errRecordReader, errInfo),
         new RowSchema(out_rwsch.getColumnInfos()), input), out_rwsch);
 
     return output;
@@ -2038,7 +2041,7 @@ public class SemanticAnalyzer extends Ba
   private List<Integer> getGroupingSetsForRollup(int size) {
     List<Integer> groupingSetKeys = new ArrayList<Integer>();
     for (int i = 0; i <= size; i++) {
-       groupingSetKeys.add((1 << i) - 1);
+      groupingSetKeys.add((1 << i) - 1);
     }
     return groupingSetKeys;
   }
@@ -2046,7 +2049,7 @@ public class SemanticAnalyzer extends Ba
   private List<Integer> getGroupingSetsForCube(int size) {
     int count = 1 << size;
     List<Integer> results = new ArrayList<Integer>(count);
-    for(int i = 0; i < count; ++i) {
+    for (int i = 0; i < count; ++i) {
       results.add(i);
     }
     return results;
@@ -2083,16 +2086,16 @@ public class SemanticAnalyzer extends Ba
     if (root != null) {
       for (int i = 0; i < root.getChildCount(); ++i) {
         ASTNode child = (ASTNode) root.getChild(i);
-        if(child.getType() != HiveParser.TOK_GROUPING_SETS_EXPRESSION) {
+        if (child.getType() != HiveParser.TOK_GROUPING_SETS_EXPRESSION) {
           continue;
         }
         int bitmap = 0;
         for (int j = 0; j < child.getChildCount(); ++j) {
           String treeAsString = child.getChild(j).toStringTree();
           Integer pos = exprPos.get(treeAsString);
-          if(pos == null) {
+          if (pos == null) {
             throw new SemanticException(
-                generateErrorMessage((ASTNode)child.getChild(j),
+                generateErrorMessage((ASTNode) child.getChild(j),
                     ErrorMsg.HIVE_GROUPING_SETS_EXPR_NOT_IN_GROUPBY.getErrorCodedMsg()));
           }
           bitmap = setBit(bitmap, pos);
@@ -2100,7 +2103,7 @@ public class SemanticAnalyzer extends Ba
         result.add(bitmap);
       }
     }
-    if(checkForNoAggr(result)) {
+    if (checkForNoAggr(result)) {
       throw new SemanticException(
           ErrorMsg.HIVE_GROUPING_SETS_AGGR_NOFUNC.getMsg());
     }
@@ -2109,7 +2112,7 @@ public class SemanticAnalyzer extends Ba
 
   private boolean checkForNoAggr(List<Integer> bitmaps) {
     boolean ret = true;
-    for(int mask : bitmaps) {
+    for (int mask : bitmaps) {
       ret &= mask == 0;
     }
     return ret;
@@ -2173,7 +2176,7 @@ public class SemanticAnalyzer extends Ba
     ASTNode root = (ASTNode) selExpr.getChild(0);
     if (root.getType() == HiveParser.TOK_TABLE_OR_COL) {
       colAlias =
-        BaseSemanticAnalyzer.unescapeIdentifier(root.getChild(0).getText());
+          BaseSemanticAnalyzer.unescapeIdentifier(root.getChild(0).getText());
       colRef[0] = tabAlias;
       colRef[1] = colAlias;
       return colRef;
@@ -2195,23 +2198,23 @@ public class SemanticAnalyzer extends Ba
       }
     }
 
-    //if specified generate alias using func name
+    // if specified generate alias using func name
     if (includeFuncName && (root.getType() == HiveParser.TOK_FUNCTION)) {
 
       String expr_flattened = root.toStringTree();
 
-      //remove all TOK tokens
+      // remove all TOK tokens
       String expr_no_tok = expr_flattened.replaceAll("TOK_\\S+", "");
 
-      //remove all non alphanumeric letters, replace whitespace spans with underscore
-      String  expr_formatted = expr_no_tok.replaceAll("\\W", " ").trim().replaceAll("\\s+", "_");
+      // remove all non alphanumeric letters, replace whitespace spans with underscore
+      String expr_formatted = expr_no_tok.replaceAll("\\W", " ").trim().replaceAll("\\s+", "_");
 
-      //limit length to 20 chars
-      if(expr_formatted.length()>AUTOGEN_COLALIAS_PRFX_MAXLENGTH) {
+      // limit length to 20 chars
+      if (expr_formatted.length() > AUTOGEN_COLALIAS_PRFX_MAXLENGTH) {
         expr_formatted = expr_formatted.substring(0, AUTOGEN_COLALIAS_PRFX_MAXLENGTH);
       }
 
-      //append colnum to make it unique
+      // append colnum to make it unique
       colAlias = expr_formatted.concat("_" + colNum);
     }
 
@@ -2274,7 +2277,7 @@ public class SemanticAnalyzer extends Ba
     }
 
     boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() ==
-      HiveParser.TOK_TRANSFORM);
+        HiveParser.TOK_TRANSFORM);
     if (isInTransform) {
       queryProperties.setUsesScript(true);
       globalLimitCtx.setHasTransformOrUDTF(true);
@@ -2314,14 +2317,14 @@ public class SemanticAnalyzer extends Ba
       // Only support a single expression when it's a UDTF
       if (selExprList.getChildCount() > 1) {
         throw new SemanticException(generateErrorMessage(
-                    (ASTNode) selExprList.getChild(1),
-                    ErrorMsg.UDTF_MULTIPLE_EXPR.getMsg()));
+            (ASTNode) selExprList.getChild(1),
+            ErrorMsg.UDTF_MULTIPLE_EXPR.getMsg()));
       }
       // Require an AS for UDTFs for column aliases
       ASTNode selExpr = (ASTNode) selExprList.getChild(posn);
       if (selExpr.getChildCount() < 2) {
         throw new SemanticException(generateErrorMessage(udtfExpr,
-                    ErrorMsg.UDTF_REQUIRE_AS.getMsg()));
+            ErrorMsg.UDTF_REQUIRE_AS.getMsg()));
       }
       // Get the column / table aliases from the expression. Start from 1 as
       // 0 is the TOK_FUNCTION
@@ -2382,8 +2385,8 @@ public class SemanticAnalyzer extends Ba
       // AST's are slightly different.
       if (!isInTransform && !isUDTF && child.getChildCount() > 2) {
         throw new SemanticException(generateErrorMessage(
-                    (ASTNode) child.getChild(2),
-                    ErrorMsg.INVALID_AS.getMsg()));
+            (ASTNode) child.getChild(2),
+            ErrorMsg.INVALID_AS.getMsg()));
       }
 
       // The real expression
@@ -2399,7 +2402,7 @@ public class SemanticAnalyzer extends Ba
         // Get rid of TOK_SELEXPR
         expr = (ASTNode) child.getChild(0);
         String[] colRef = getColAlias(child, autogenColAliasPrfxLbl, inputRR,
-                                             autogenColAliasPrfxIncludeFuncName, i);
+            autogenColAliasPrfxIncludeFuncName, i);
         tabAlias = colRef[0];
         colAlias = colRef[1];
         if (hasAsClause) {
@@ -2452,9 +2455,9 @@ public class SemanticAnalyzer extends Ba
         }
 
         ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(pos),
-          exp.getWritableObjectInspector(), tabAlias, false);
+            exp.getWritableObjectInspector(), tabAlias, false);
         colInfo.setSkewedCol((exp instanceof ExprNodeColumnDesc) ? ((ExprNodeColumnDesc) exp)
-          .isSkewedCol() : false);
+            .isSkewedCol() : false);
         out_rwsch.put(tabAlias, colAlias, colInfo);
 
         pos = Integer.valueOf(pos.intValue() + 1);
@@ -2477,7 +2480,7 @@ public class SemanticAnalyzer extends Ba
 
     Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(
         new SelectDesc(col_list, columnNames, selectStar), new RowSchema(
-        out_rwsch.getColumnInfos()), input), out_rwsch);
+            out_rwsch.getColumnInfos()), input), out_rwsch);
 
     output.setColumnExprMap(colExprMap);
     if (isInTransform) {
@@ -2557,7 +2560,7 @@ public class SemanticAnalyzer extends Ba
       boolean isDistinct, boolean isAllColumns)
       throws SemanticException {
     ArrayList<ObjectInspector> originalParameterTypeInfos =
-      getWritableObjectInspector(aggParameters);
+        getWritableObjectInspector(aggParameters);
     GenericUDAFEvaluator result = FunctionRegistry.getGenericUDAFEvaluator(
         aggName, originalParameterTypeInfos, isDistinct, isAllColumns);
     if (null == result) {
@@ -2733,9 +2736,9 @@ public class SemanticAnalyzer extends Ba
       List<String> inputKeyCols = ((ReduceSinkDesc)
           reduceSinkOperatorInfo.getConf()).getOutputKeyColumnNames();
       if (inputKeyCols.size() > 0) {
-        lastKeyColName = inputKeyCols.get(inputKeyCols.size()-1);
+        lastKeyColName = inputKeyCols.get(inputKeyCols.size() - 1);
       }
-      reduceValues = ((ReduceSinkDesc)reduceSinkOperatorInfo.getConf()).getValueCols();
+      reduceValues = ((ReduceSinkDesc) reduceSinkOperatorInfo.getConf()).getValueCols();
     }
     int numDistinctUDFs = 0;
     for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
@@ -2752,7 +2755,7 @@ public class SemanticAnalyzer extends Ba
       for (int i = 1; i < value.getChildCount(); i++) {
         ASTNode paraExpr = (ASTNode) value.getChild(i);
         ColumnInfo paraExprInfo =
-          groupByInputRowResolver.getExpression(paraExpr);
+            groupByInputRowResolver.getExpression(paraExpr);
         if (paraExprInfo == null) {
           throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg(paraExpr));
         }
@@ -2763,8 +2766,8 @@ public class SemanticAnalyzer extends Ba
           // if aggr is distinct, the parameter is name is constructed as
           // KEY.lastKeyColName:<tag>._colx
           paraExpression = Utilities.ReduceField.KEY.name() + "." +
-          lastKeyColName + ":" + numDistinctUDFs + "." +
-          getColumnInternalName(i-1);
+              lastKeyColName + ":" + numDistinctUDFs + "." +
+              getColumnInternalName(i - 1);
 
         }
 
@@ -2806,13 +2809,14 @@ public class SemanticAnalyzer extends Ba
       }
     }
     float groupByMemoryUsage = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
-    float memoryThreshold = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
+    float memoryThreshold = HiveConf
+        .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
 
     Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild(
-      new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
-      false,groupByMemoryUsage,memoryThreshold, null, false, 0),
-      new RowSchema(groupByOutputRowResolver.getColumnInfos()),
-      reduceSinkOperatorInfo), groupByOutputRowResolver);
+        new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
+            false, groupByMemoryUsage, memoryThreshold, null, false, 0),
+        new RowSchema(groupByOutputRowResolver.getColumnInfos()),
+        reduceSinkOperatorInfo), groupByOutputRowResolver);
     op.setColumnExprMap(colExprMap);
     return op;
   }
@@ -2823,7 +2827,7 @@ public class SemanticAnalyzer extends Ba
   // For eg: consider: select key, value, count(1) from T group by key, value with rollup.
   // Assuming map-side aggregation and no skew, the plan would look like:
   //
-  //   TableScan --> Select --> GroupBy1 --> ReduceSink --> GroupBy2 --> Select --> FileSink
+  // TableScan --> Select --> GroupBy1 --> ReduceSink --> GroupBy2 --> Select --> FileSink
   //
   // This function is called for GroupBy2 to pass the additional grouping keys introduced by
   // GroupBy1 for the grouping set (corresponding to the rollup).
@@ -2834,9 +2838,9 @@ public class SemanticAnalyzer extends Ba
       Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
     // For grouping sets, add a dummy grouping key
     String groupingSetColumnName =
-      groupByInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName();
+        groupByInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName();
     ExprNodeDesc inputExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
-      groupingSetColumnName, null, false);
+        groupingSetColumnName, null, false);
     groupByKeys.add(inputExpr);
 
     String field = getColumnInternalName(groupByKeys.size() - 1);
@@ -2854,7 +2858,7 @@ public class SemanticAnalyzer extends Ba
   // For eg: consider: select key, value, count(1) from T group by key, value with rollup.
   // Assuming map-side aggregation and no skew, the plan would look like:
   //
-  //   TableScan --> Select --> GroupBy1 --> ReduceSink --> GroupBy2 --> Select --> FileSink
+  // TableScan --> Select --> GroupBy1 --> ReduceSink --> GroupBy2 --> Select --> FileSink
   //
   // This function is called for ReduceSink to add the additional grouping keys introduced by
   // GroupBy1 into the reduce keys.
@@ -2865,16 +2869,16 @@ public class SemanticAnalyzer extends Ba
       Map<String, ExprNodeDesc> colExprMap) throws SemanticException {
     // add a key for reduce sink
     String groupingSetColumnName =
-      reduceSinkInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName();
+        reduceSinkInputRowResolver.get(null, VirtualColumn.GROUPINGID.getName()).getInternalName();
     ExprNodeDesc inputExpr = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo,
-      groupingSetColumnName, null, false);
+        groupingSetColumnName, null, false);
     reduceKeys.add(inputExpr);
 
     outputKeyColumnNames.add(getColumnInternalName(reduceKeys.size() - 1));
     String field = Utilities.ReduceField.KEY.toString() + "."
-      + getColumnInternalName(reduceKeys.size() - 1);
+        + getColumnInternalName(reduceKeys.size() - 1);
     ColumnInfo colInfo = new ColumnInfo(field, reduceKeys.get(
-      reduceKeys.size() - 1).getTypeInfo(), null, true);
+        reduceKeys.size() - 1).getTypeInfo(), null, true);
     reduceSinkOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(), colInfo);
     colExprMap.put(colInfo.getInternalName(), inputExpr);
   }
@@ -2891,6 +2895,12 @@ public class SemanticAnalyzer extends Ba
    *          genericUDAFEvaluator.
    * @param distPartAggr
    *          partial aggregation for distincts
+   * @param groupingSets
+   *          list of grouping sets
+   * @param groupingSetsPresent
+   *          whether grouping sets are present in this query
+   * @param groupingSetsConsumedCurrentMR
+   *          whether grouping sets are consumed by this group by
    * @return the new GroupByOperator
    */
   @SuppressWarnings("nls")
@@ -2898,7 +2908,9 @@ public class SemanticAnalyzer extends Ba
       String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode,
       Map<String, GenericUDAFEvaluator> genericUDAFEvaluators,
       boolean distPartAgg,
-      boolean groupingSetsPresent) throws SemanticException {
+      List<Integer> groupingSets,
+      boolean groupingSetsPresent,
+      boolean groupingSetsNeedAdditionalMRJob) throws SemanticException {
     ArrayList<String> outputColumnNames = new ArrayList<String>();
     RowResolver groupByInputRowResolver = opParseCtx
         .get(reduceSinkOperatorInfo).getRowResolver();
@@ -2926,14 +2938,39 @@ public class SemanticAnalyzer extends Ba
       colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1));
     }
 
+    // This is only needed if a new grouping set key is being created
+    int groupingSetsPosition = 0;
+
     // For grouping sets, add a dummy grouping key
     if (groupingSetsPresent) {
-      addGroupingSetKey(
-        groupByKeys,
-        groupByInputRowResolver,
-        groupByOutputRowResolver,
-        outputColumnNames,
-        colExprMap);
+      // Consider the query: select a,b, count(1) from T group by a,b with cube;
+      // where it is being executed in a single map-reduce job
+      // The plan is TableScan -> GroupBy1 -> ReduceSink -> GroupBy2 -> FileSink
+      // GroupBy1 already added the grouping id as part of the row
+      // This function is called for GroupBy2 to add grouping id as part of the groupby keys
+      if (!groupingSetsNeedAdditionalMRJob) {
+        addGroupingSetKey(
+            groupByKeys,
+            groupByInputRowResolver,
+            groupByOutputRowResolver,
+            outputColumnNames,
+            colExprMap);
+      }
+      else {
+        groupingSetsPosition = groupByKeys.size();
+        // The grouping set has not yet been processed. Create a new grouping key
+        // Consider the query: select a,b, count(1) from T group by a,b with cube;
+        // where it is being executed in 2 map-reduce jobs
+        // The plan for 1st MR is TableScan -> GroupBy1 -> ReduceSink -> GroupBy2 -> FileSink
+        // GroupBy1/ReduceSink worked as if grouping sets were not present
+        // This function is called for GroupBy2 to create new rows for grouping sets
+        // For each input row (a,b), 4 rows are created for the example above:
+        // (a,b), (a,null), (null, b), (null, null)
+        createNewGroupingKey(groupByKeys,
+            outputColumnNames,
+            groupByOutputRowResolver,
+            colExprMap);
+      }
     }
 
     HashMap<String, ASTNode> aggregationTrees = parseInfo
@@ -2946,9 +2983,9 @@ public class SemanticAnalyzer extends Ba
       List<String> inputKeyCols = ((ReduceSinkDesc)
           reduceSinkOperatorInfo.getConf()).getOutputKeyColumnNames();
       if (inputKeyCols.size() > 0) {
-        lastKeyColName = inputKeyCols.get(inputKeyCols.size()-1);
+        lastKeyColName = inputKeyCols.get(inputKeyCols.size() - 1);
       }
-      reduceValues = ((ReduceSinkDesc)reduceSinkOperatorInfo.getConf()).getValueCols();
+      reduceValues = ((ReduceSinkDesc) reduceSinkOperatorInfo.getConf()).getValueCols();
     }
     int numDistinctUDFs = 0;
     for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
@@ -2973,7 +3010,7 @@ public class SemanticAnalyzer extends Ba
         for (int i = 1; i < value.getChildCount(); i++) {
           ASTNode paraExpr = (ASTNode) value.getChild(i);
           ColumnInfo paraExprInfo =
-            groupByInputRowResolver.getExpression(paraExpr);
+              groupByInputRowResolver.getExpression(paraExpr);
           if (paraExprInfo == null) {
             throw new SemanticException(ErrorMsg.INVALID_COLUMN
                 .getMsg(paraExpr));
@@ -2985,8 +3022,8 @@ public class SemanticAnalyzer extends Ba
             // if aggr is distinct, the parameter is name is constructed as
             // KEY.lastKeyColName:<tag>._colx
             paraExpression = Utilities.ReduceField.KEY.name() + "." +
-            lastKeyColName + ":" + numDistinctUDFs + "."
-            + getColumnInternalName(i-1);
+                lastKeyColName + ":" + numDistinctUDFs + "."
+                + getColumnInternalName(i - 1);
 
           }
 
@@ -3013,7 +3050,7 @@ public class SemanticAnalyzer extends Ba
         assert (paraExpression != null);
         aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(),
             paraExpression, paraExprInfo.getTabAlias(), paraExprInfo
-            .getIsVirtualCol()));
+                .getIsVirtualCol()));
       }
       if (isDistinct) {
         numDistinctUDFs++;
@@ -3044,20 +3081,50 @@ public class SemanticAnalyzer extends Ba
           field, udaf.returnType, "", false));
     }
     float groupByMemoryUsage = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
-    float memoryThreshold = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
+    float memoryThreshold = HiveConf
+        .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
 
-    // Nothing special needs to be done for grouping sets.
-    // This is the final group by operator, so multiple rows corresponding to the
+    // Nothing special needs to be done for grouping sets if
+    // this is the final group by operator, and multiple rows corresponding to the
     // grouping sets have been generated upstream.
+    // However, if an addition MR job has been created to handle grouping sets,
+    // additional rows corresponding to grouping sets need to be created here.
     Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild(
-      new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
-      distPartAgg,groupByMemoryUsage,memoryThreshold, null, false, 0),
-      new RowSchema(groupByOutputRowResolver.getColumnInfos()), reduceSinkOperatorInfo),
-      groupByOutputRowResolver);
+        new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
+            distPartAgg, groupByMemoryUsage, memoryThreshold,
+            groupingSets,
+            groupingSetsPresent && groupingSetsNeedAdditionalMRJob,
+            groupingSetsPosition),
+        new RowSchema(groupByOutputRowResolver.getColumnInfos()), reduceSinkOperatorInfo),
+        groupByOutputRowResolver);
     op.setColumnExprMap(colExprMap);
     return op;
   }
 
+  /*
+   * Create a new grouping key for grouping id.
+   * A dummy grouping id. is added. At runtime, the group by operator
+   * creates 'n' rows per input row, where 'n' is the number of grouping sets.
+   */
+  private void createNewGroupingKey(List<ExprNodeDesc> groupByKeys,
+      List<String> outputColumnNames,
+      RowResolver groupByOutputRowResolver,
+      Map<String, ExprNodeDesc> colExprMap) {
+    // The value for the constant does not matter. It is replaced by the grouping set
+    // value for the actual implementation
+    ExprNodeConstantDesc constant = new ExprNodeConstantDesc("0");
+    groupByKeys.add(constant);
+    String field = getColumnInternalName(groupByKeys.size() - 1);
+    outputColumnNames.add(field);
+    groupByOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(),
+        new ColumnInfo(
+            field,
+            TypeInfoFactory.stringTypeInfo,
+            null,
+            true));
+    colExprMap.put(field, constant);
+  }
+
   /**
    * Generate the map-side GroupByOperator for the Query Block
    * (qb.getParseInfo().getXXX(dest)). The new GroupByOperator will be a child
@@ -3093,13 +3160,13 @@ public class SemanticAnalyzer extends Ba
     for (int i = 0; i < grpByExprs.size(); ++i) {
       ASTNode grpbyExpr = grpByExprs.get(i);
       ExprNodeDesc grpByExprNode = genExprNodeDesc(grpbyExpr,
-        groupByInputRowResolver);
+          groupByInputRowResolver);
 
       groupByKeys.add(grpByExprNode);
       String field = getColumnInternalName(i);
       outputColumnNames.add(field);
       groupByOutputRowResolver.putExpression(grpbyExpr,
-        new ColumnInfo(field, grpByExprNode.getTypeInfo(), "", false));
+          new ColumnInfo(field, grpByExprNode.getTypeInfo(), "", false));
       colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1));
     }
 
@@ -3111,48 +3178,38 @@ public class SemanticAnalyzer extends Ba
     // For eg: consider: select key, value, count(1) from T group by key, value with rollup.
     // Assuming map-side aggregation and no skew, the plan would look like:
     //
-    //   TableScan --> Select --> GroupBy1 --> ReduceSink --> GroupBy2 --> Select --> FileSink
+    // TableScan --> Select --> GroupBy1 --> ReduceSink --> GroupBy2 --> Select --> FileSink
     //
     // This function is called for GroupBy1 to create an additional grouping key
     // for the grouping set (corresponding to the rollup).
     if (groupingSetsPresent) {
-      // The value for the constant does not matter. It is replaced by the grouping set
-      // value for the actual implementation
-      ExprNodeConstantDesc constant = new ExprNodeConstantDesc("0");
-      groupByKeys.add(constant);
-      String field = getColumnInternalName(groupByKeys.size() - 1);
-      outputColumnNames.add(field);
-      groupByOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(),
-        new ColumnInfo(
-            field,
-            TypeInfoFactory.stringTypeInfo,
-            null,
-            true));
-      colExprMap.put(field, constant);
+      createNewGroupingKey(groupByKeys,
+          outputColumnNames,
+          groupByOutputRowResolver,
+          colExprMap);
     }
 
     // If there is a distinctFuncExp, add all parameters to the reduceKeys.
     if (!parseInfo.getDistinctFuncExprsForClause(dest).isEmpty()) {
       List<ASTNode> list = parseInfo.getDistinctFuncExprsForClause(dest);
-      for(ASTNode value: list) {
+      for (ASTNode value : list) {
         // 0 is function name
         for (int i = 1; i < value.getChildCount(); i++) {
           ASTNode parameter = (ASTNode) value.getChild(i);
           if (groupByOutputRowResolver.getExpression(parameter) == null) {
             ExprNodeDesc distExprNode = genExprNodeDesc(parameter,
-              groupByInputRowResolver);
+                groupByInputRowResolver);
             groupByKeys.add(distExprNode);
-            String field = getColumnInternalName(groupByKeys.size()-1);
+            String field = getColumnInternalName(groupByKeys.size() - 1);
             outputColumnNames.add(field);
             groupByOutputRowResolver.putExpression(parameter, new ColumnInfo(
-              field, distExprNode.getTypeInfo(), "", false));
+                field, distExprNode.getTypeInfo(), "", false));
             colExprMap.put(field, groupByKeys.get(groupByKeys.size() - 1));
           }
         }
       }
     }
 
-
     // For each aggregation
     HashMap<String, ASTNode> aggregationTrees = parseInfo
         .getAggregationExprsForClause(dest);
@@ -3167,7 +3224,7 @@ public class SemanticAnalyzer extends Ba
       for (int i = 1; i < value.getChildCount(); i++) {
         ASTNode paraExpr = (ASTNode) value.getChild(i);
         ExprNodeDesc paraExprNode = genExprNodeDesc(paraExpr,
-          groupByInputRowResolver);
+            groupByInputRowResolver);
 
         aggParameters.add(paraExprNode);
       }
@@ -3196,13 +3253,14 @@ public class SemanticAnalyzer extends Ba
       }
     }
     float groupByMemoryUsage = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
-    float memoryThreshold = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
+    float memoryThreshold = HiveConf
+        .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
     Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild(
-      new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
-      false,groupByMemoryUsage,memoryThreshold,
-      groupingSetKeys, groupingSetsPresent, groupingSetsPosition),
-      new RowSchema(groupByOutputRowResolver.getColumnInfos()),
-      inputOperatorInfo), groupByOutputRowResolver);
+        new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
+            false, groupByMemoryUsage, memoryThreshold,
+            groupingSetKeys, groupingSetsPresent, groupingSetsPosition),
+        new RowSchema(groupByOutputRowResolver.getColumnInfos()),
+        inputOperatorInfo), groupByOutputRowResolver);
     op.setColumnExprMap(colExprMap);
     return op;
   }
@@ -3244,18 +3302,18 @@ public class SemanticAnalyzer extends Ba
     List<String> outputValueColumnNames = new ArrayList<String>();
 
     ArrayList<ExprNodeDesc> reduceKeys = getReduceKeysForReduceSink(grpByExprs, dest,
-      reduceSinkInputRowResolver, reduceSinkOutputRowResolver, outputKeyColumnNames,
-      colExprMap);
+        reduceSinkInputRowResolver, reduceSinkOutputRowResolver, outputKeyColumnNames,
+        colExprMap);
 
     // add a key for reduce sink
     if (groupingSetsPresent) {
       // Process grouping set for the reduce sink operator
       processGroupingSetReduceSinkOperator(
-        reduceSinkInputRowResolver,
-        reduceSinkOutputRowResolver,
-        reduceKeys,
-        outputKeyColumnNames,
-        colExprMap);
+          reduceSinkInputRowResolver,
+          reduceSinkOutputRowResolver,
+          reduceKeys,
+          outputKeyColumnNames,
+          colExprMap);
 
       if (changeNumPartitionFields) {
         numPartitionFields++;
@@ -3292,14 +3350,14 @@ public class SemanticAnalyzer extends Ba
     }
 
     ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap(
-      OperatorFactory.getAndMakeChild(
-      PlanUtils.getReduceSinkDesc(reduceKeys,
-      groupingSetsPresent ? grpByExprs.size() + 1 : grpByExprs.size(),
-      reduceValues, distinctColIndices,
-      outputKeyColumnNames, outputValueColumnNames, true, -1, numPartitionFields,
-      numReducers),
-      new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), inputOperatorInfo),
-      reduceSinkOutputRowResolver);
+        OperatorFactory.getAndMakeChild(
+            PlanUtils.getReduceSinkDesc(reduceKeys,
+                groupingSetsPresent ? grpByExprs.size() + 1 : grpByExprs.size(),
+                reduceValues, distinctColIndices,
+                outputKeyColumnNames, outputValueColumnNames, true, -1, numPartitionFields,
+                numReducers),
+            new RowSchema(reduceSinkOutputRowResolver.getColumnInfos()), inputOperatorInfo),
+        reduceSinkOutputRowResolver);
     rsOp.setColumnExprMap(colExprMap);
     return rsOp;
   }
@@ -3314,7 +3372,7 @@ public class SemanticAnalyzer extends Ba
     for (int i = 0; i < grpByExprs.size(); ++i) {
       ASTNode grpbyExpr = grpByExprs.get(i);
       ExprNodeDesc inputExpr = genExprNodeDesc(grpbyExpr,
-        reduceSinkInputRowResolver);
+          reduceSinkInputRowResolver);
       reduceKeys.add(inputExpr);
       if (reduceSinkOutputRowResolver.getExpression(grpbyExpr) == null) {
         outputKeyColumnNames.add(getColumnInternalName(reduceKeys.size() - 1));
@@ -3333,7 +3391,8 @@ public class SemanticAnalyzer extends Ba
     return reduceKeys;
   }
 
-  private List<List<Integer>> getDistinctColIndicesForReduceSink(QBParseInfo parseInfo, String dest,
+  private List<List<Integer>> getDistinctColIndicesForReduceSink(QBParseInfo parseInfo,
+      String dest,
       List<ExprNodeDesc> reduceKeys, RowResolver reduceSinkInputRowResolver,
       RowResolver reduceSinkOutputRowResolver, List<String> outputKeyColumnNames)
       throws SemanticException {
@@ -3369,8 +3428,8 @@ public class SemanticAnalyzer extends Ba
           distinctIndices.add(ri);
           String name = getColumnInternalName(numExprs);
           String field = Utilities.ReduceField.KEY.toString() + "." + colName
-          + ":" + i
-          + "." + name;
+              + ":" + i
+              + "." + name;
           ColumnInfo colInfo = new ColumnInfo(field, expr.getTypeInfo(), null, false);
           reduceSinkOutputRowResolver.putExpression(parameter, colInfo);
           numExprs++;
@@ -3470,10 +3529,10 @@ public class SemanticAnalyzer extends Ba
 
     ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap(
         OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys,
-        grpByExprs.size(), reduceValues, distinctColIndices,
-        outputKeyColumnNames, outputValueColumnNames, true, -1, grpByExprs.size(),
-        -1), new RowSchema(reduceSinkOutputRowResolver
-        .getColumnInfos()), inputOperatorInfo), reduceSinkOutputRowResolver);
+            grpByExprs.size(), reduceValues, distinctColIndices,
+            outputKeyColumnNames, outputValueColumnNames, true, -1, grpByExprs.size(),
+            -1), new RowSchema(reduceSinkOutputRowResolver
+            .getColumnInfos()), inputOperatorInfo), reduceSinkOutputRowResolver);
     rsOp.setColumnExprMap(colExprMap);
     return rsOp;
   }
@@ -3494,10 +3553,10 @@ public class SemanticAnalyzer extends Ba
       return nodes;
     }
     for (int i = 0; i < node.getChildCount(); i++) {
-      ASTNode child = (ASTNode)node.getChild(i);
+      ASTNode child = (ASTNode) node.getChild(i);
       if (child.getType() == HiveParser.TOK_TABLE_OR_COL && child.getChild(0) != null &&
           inputRR.get(null,
-                  BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText())) != null) {
+              BaseSemanticAnalyzer.unescapeIdentifier(child.getChild(0).getText())) != null) {
         nodes.add(child);
       } else {
         nodes.addAll(getColumnExprsFromASTNode(child, inputRR));
@@ -3558,11 +3617,11 @@ public class SemanticAnalyzer extends Ba
       // Note that partitioning fields dont need to change, since it is either
       // partitioned randomly, or by all grouping keys + distinct keys
       processGroupingSetReduceSinkOperator(
-        reduceSinkInputRowResolver2,
-        reduceSinkOutputRowResolver2,
-        reduceKeys,
-        outputColumnNames,
-        colExprMap);
+          reduceSinkInputRowResolver2,
+          reduceSinkOutputRowResolver2,
+          reduceKeys,
+          outputColumnNames,
+          colExprMap);
     }
 
     // Get partial aggregation results and store in reduceValues
@@ -3586,9 +3645,9 @@ public class SemanticAnalyzer extends Ba
 
     ReduceSinkOperator rsOp = (ReduceSinkOperator) putOpInsertMap(
         OperatorFactory.getAndMakeChild(PlanUtils.getReduceSinkDesc(reduceKeys,
-        reduceValues, outputColumnNames, true, -1, numPartitionFields,
-        numReducers), new RowSchema(reduceSinkOutputRowResolver2
-        .getColumnInfos()), groupByOperatorInfo),
+            reduceValues, outputColumnNames, true, -1, numPartitionFields,
+            numReducers), new RowSchema(reduceSinkOutputRowResolver2
+            .getColumnInfos()), groupByOperatorInfo),
         reduceSinkOutputRowResolver2);
 
     rsOp.setColumnExprMap(colExprMap);
@@ -3645,11 +3704,11 @@ public class SemanticAnalyzer extends Ba
     // For grouping sets, add a dummy grouping key
     if (groupingSetsPresent) {
       addGroupingSetKey(
-        groupByKeys,
-        groupByInputRowResolver2,
-        groupByOutputRowResolver2,
-        outputColumnNames,
-        colExprMap);
+          groupByKeys,
+          groupByInputRowResolver2,
+          groupByOutputRowResolver2,
+          outputColumnNames,
+          colExprMap);
     }
 
     HashMap<String, ASTNode> aggregationTrees = parseInfo
@@ -3665,7 +3724,7 @@ public class SemanticAnalyzer extends Ba
       assert (paraExpression != null);
       aggParameters.add(new ExprNodeColumnDesc(paraExprInfo.getType(),
           paraExpression, paraExprInfo.getTabAlias(), paraExprInfo
-          .getIsVirtualCol()));
+              .getIsVirtualCol()));
 
       String aggName = unescapeIdentifier(value.getChild(0).getText());
 
@@ -3683,7 +3742,7 @@ public class SemanticAnalyzer extends Ba
               udaf.genericUDAFEvaluator,
               udaf.convertedParameters,
               (mode != GroupByDesc.Mode.FINAL && value.getToken().getType() ==
-                HiveParser.TOK_FUNCTIONDI),
+              HiveParser.TOK_FUNCTIONDI),
               amode));
       String field = getColumnInternalName(groupByKeys.size()
           + aggregations.size() - 1);
@@ -3692,13 +3751,14 @@ public class SemanticAnalyzer extends Ba
           field, udaf.returnType, "", false));
     }
     float groupByMemoryUsage = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRHASHMEMORY);
-    float memoryThreshold = HiveConf.getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
+    float memoryThreshold = HiveConf
+        .getFloatVar(conf, HiveConf.ConfVars.HIVEMAPAGGRMEMORYTHRESHOLD);
 
     Operator op = putOpInsertMap(OperatorFactory.getAndMakeChild(
-      new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
-      false,groupByMemoryUsage,memoryThreshold, null, false, 0),
-      new RowSchema(groupByOutputRowResolver2.getColumnInfos()),
-      reduceSinkOperatorInfo2), groupByOutputRowResolver2);
+        new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
+            false, groupByMemoryUsage, memoryThreshold, null, false, 0),
+        new RowSchema(groupByOutputRowResolver2.getColumnInfos()),
+        reduceSinkOperatorInfo2), groupByOutputRowResolver2);
     op.setColumnExprMap(colExprMap);
     return op;
   }
@@ -3736,7 +3796,7 @@ public class SemanticAnalyzer extends Ba
 
     int numReducers = -1;
     ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets =
-      getGroupByGroupingSetsForClause(parseInfo, dest);
+        getGroupByGroupingSetsForClause(parseInfo, dest);
 
     List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst();
     List<Integer> groupingSets = grpByExprsGroupingSets.getSecond();
@@ -3752,15 +3812,15 @@ public class SemanticAnalyzer extends Ba
 
     // ////// 1. Generate ReduceSinkOperator
     Operator reduceSinkOperatorInfo =
-      genGroupByPlanReduceSinkOperator(qb,
-        dest,
-        input,
-        grpByExprs,
-        grpByExprs.size(),
-        false,
-        numReducers,
-        false,
-        false);
+        genGroupByPlanReduceSinkOperator(qb,
+            dest,
+            input,
+            grpByExprs,
+            grpByExprs.size(),
+            false,
+            numReducers,
+            false,
+            false);
 
     // ////// 2. Generate GroupbyOperator
     Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo,
@@ -3788,7 +3848,7 @@ public class SemanticAnalyzer extends Ba
       if (whereExpr != null) {
         OpParseContext inputCtx = opParseCtx.get(input);
         RowResolver inputRR = inputCtx.getRowResolver();
-        ExprNodeDesc current = genExprNodeDesc((ASTNode)whereExpr.getChild(0), inputRR);
+        ExprNodeDesc current = genExprNodeDesc((ASTNode) whereExpr.getChild(0), inputRR);
 
         // Check the list of where expressions already added so they aren't duplicated
         ExprNodeDesc.ExprNodeDescEqualityWrapper currentWrapped =
@@ -3825,8 +3885,8 @@ public class SemanticAnalyzer extends Ba
       FilterDesc orFilterDesc = new FilterDesc(previous, false);
 
       selectInput = putOpInsertMap(OperatorFactory.getAndMakeChild(
-        orFilterDesc, new RowSchema(
-        inputRR.getColumnInfos()), input), inputRR);
+          orFilterDesc, new RowSchema(
+              inputRR.getColumnInfos()), input), inputRR);
     }
 
     // insert a select operator here used by the ColumnPruner to reduce
@@ -3905,20 +3965,21 @@ public class SemanticAnalyzer extends Ba
 
     // ////// Generate GroupbyOperator for a map-side partial aggregation
     Map<String, GenericUDAFEvaluator> genericUDAFEvaluators =
-      new LinkedHashMap<String, GenericUDAFEvaluator>();
+        new LinkedHashMap<String, GenericUDAFEvaluator>();
 
     QBParseInfo parseInfo = qb.getParseInfo();
 
     // ////// 2. Generate GroupbyOperator
     Operator groupByOperatorInfo = genGroupByPlanGroupByOperator1(parseInfo,
-        dest, input, GroupByDesc.Mode.HASH, genericUDAFEvaluators, true, false);
+        dest, input, GroupByDesc.Mode.HASH, genericUDAFEvaluators, true,
+        null, false, false);
 
     int numReducers = -1;
     List<ASTNode> grpByExprs = getGroupByForClause(parseInfo, dest);
 
     // ////// 3. Generate ReduceSinkOperator2
     Operator reduceSinkOperatorInfo2 = genGroupByPlanReduceSinkOperator2MR(
-      parseInfo, dest, groupByOperatorInfo, grpByExprs.size(), numReducers, false);
+        parseInfo, dest, groupByOperatorInfo, grpByExprs.size(), numReducers, false);
 
     // ////// 4. Generate GroupbyOperator2
     Operator groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR(parseInfo,
@@ -3976,7 +4037,7 @@ public class SemanticAnalyzer extends Ba
     QBParseInfo parseInfo = qb.getParseInfo();
 
     ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets =
-      getGroupByGroupingSetsForClause(parseInfo, dest);
+        getGroupByGroupingSetsForClause(parseInfo, dest);
 
     List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst();
     List<Integer> groupingSets = grpByExprsGroupingSets.getSecond();
@@ -3996,22 +4057,22 @@ public class SemanticAnalyzer extends Ba
     // operator. We set the numPartitionColumns to -1 for this purpose. This is
     // captured by WritableComparableHiveObject.hashCode() function.
     Operator reduceSinkOperatorInfo =
-      genGroupByPlanReduceSinkOperator(qb,
-        dest,
-        input,
-        grpByExprs,
-        (parseInfo.getDistinctFuncExprsForClause(dest).isEmpty() ? -1 : Integer.MAX_VALUE),
-        false,
-        -1,
-        false,
-        false);
+        genGroupByPlanReduceSinkOperator(qb,
+            dest,
+            input,
+            grpByExprs,
+            (parseInfo.getDistinctFuncExprsForClause(dest).isEmpty() ? -1 : Integer.MAX_VALUE),
+            false,
+            -1,
+            false,
+            false);
 
     // ////// 2. Generate GroupbyOperator
     Map<String, GenericUDAFEvaluator> genericUDAFEvaluators =
-      new LinkedHashMap<String, GenericUDAFEvaluator>();
+        new LinkedHashMap<String, GenericUDAFEvaluator>();
     GroupByOperator groupByOperatorInfo = (GroupByOperator) genGroupByPlanGroupByOperator(
-      parseInfo, dest, reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIAL1,
-      genericUDAFEvaluators);
+        parseInfo, dest, reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIAL1,
+        genericUDAFEvaluators);
 
     int numReducers = -1;
     if (grpByExprs.isEmpty()) {
@@ -4020,12 +4081,12 @@ public class SemanticAnalyzer extends Ba
 
     // ////// 3. Generate ReduceSinkOperator2
     Operator reduceSinkOperatorInfo2 = genGroupByPlanReduceSinkOperator2MR(
-      parseInfo, dest, groupByOperatorInfo, grpByExprs.size(), numReducers, false);
+        parseInfo, dest, groupByOperatorInfo, grpByExprs.size(), numReducers, false);
 
     // ////// 4. Generate GroupbyOperator2
     Operator groupByOperatorInfo2 = genGroupByPlanGroupByOperator2MR(parseInfo,
-      dest, reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL,
-      genericUDAFEvaluators, false);
+        dest, reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL,
+        genericUDAFEvaluators, false);
 
     return groupByOperatorInfo2;
   }
@@ -4046,13 +4107,13 @@ public class SemanticAnalyzer extends Ba
   static private void extractColumns(Set<String> colNamesExprs,
       ExprNodeDesc exprNode) throws SemanticException {
     if (exprNode instanceof ExprNodeColumnDesc) {
-      colNamesExprs.add(((ExprNodeColumnDesc)exprNode).getColumn());
+      colNamesExprs.add(((ExprNodeColumnDesc) exprNode).getColumn());
       return;
     }
 
     if (exprNode instanceof ExprNodeGenericFuncDesc) {
-      ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc)exprNode;
-      for (ExprNodeDesc childExpr: funcDesc.getChildExprs()) {
+      ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) exprNode;
+      for (ExprNodeDesc childExpr : funcDesc.getChildExprs()) {
         extractColumns(colNamesExprs, childExpr);
       }
     }
@@ -4071,7 +4132,7 @@ public class SemanticAnalyzer extends Ba
   private void checkExpressionsForGroupingSet(List<ASTNode> grpByExprs,
       List<ASTNode> distinctGrpByExprs,
       Map<String, ASTNode> aggregationTrees,
-      RowResolver   inputRowResolver) throws SemanticException {
+      RowResolver inputRowResolver) throws SemanticException {
 
     Set<String> colNamesGroupByExprs = new HashSet<String>();
     Set<String> colNamesGroupByDistinctExprs = new HashSet<String>();
@@ -4085,7 +4146,7 @@ public class SemanticAnalyzer extends Ba
 
     // If there is a distinctFuncExp, add all parameters to the reduceKeys.
     if (!distinctGrpByExprs.isEmpty()) {
-      for(ASTNode value: distinctGrpByExprs) {
+      for (ASTNode value : distinctGrpByExprs) {
         // 0 is function name
         for (int i = 1; i < value.getChildCount(); i++) {
           ASTNode parameter = (ASTNode) value.getChild(i);
@@ -4095,8 +4156,7 @@ public class SemanticAnalyzer extends Ba
         }
 
         if (hasCommonElement(colNamesGroupByExprs, colNamesGroupByDistinctExprs)) {
-          throw
-            new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_EXPRESSION_INVALID.getMsg());
+          throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_EXPRESSION_INVALID.getMsg());
         }
       }
     }
@@ -4114,8 +4174,7 @@ public class SemanticAnalyzer extends Ba
       }
 
       if (hasCommonElement(colNamesGroupByExprs, colNamesAggregateParameters)) {
-        throw
-          new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_EXPRESSION_INVALID.getMsg());
+        throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_EXPRESSION_INVALID.getMsg());
       }
     }
   }
@@ -4127,52 +4186,92 @@ public class SemanticAnalyzer extends Ba
    * spray by the group by key, and sort by the distinct key (if any), and
    * compute aggregates based on actual aggregates
    *
-   * The agggregation evaluation functions are as follows: Mapper:
-   * iterate/terminatePartial (mode = HASH)
+   * The agggregation evaluation functions are as follows:
    *
+   * No grouping sets:
+   * Group By Operator:
+   * grouping keys: group by expressions if no DISTINCT
+   * grouping keys: group by expressions + distinct keys if DISTINCT
+   * Mapper: iterate/terminatePartial (mode = HASH)
    * Partitioning Key: grouping key
+   * Sorting Key: grouping key if no DISTINCT
+   * grouping + distinct key if DISTINCT
+   * Reducer: iterate/terminate if DISTINCT
+   * merge/terminate if NO DISTINCT (mode MERGEPARTIAL)
+   *
+   * Grouping Sets:
+   * Group By Operator:
+   * grouping keys: group by expressions + grouping id. if no DISTINCT
+   * grouping keys: group by expressions + grouping id. + distinct keys if DISTINCT
+   * Mapper: iterate/terminatePartial (mode = HASH)
+   * Partitioning Key: grouping key + grouping id.
+   * Sorting Key: grouping key + grouping id. if no DISTINCT
+   * grouping + grouping id. + distinct key if DISTINCT
+   * Reducer: iterate/terminate if DISTINCT
+   * merge/terminate if NO DISTINCT (mode MERGEPARTIAL)
+   *
+   * Grouping Sets with an additional MR job introduced (distincts are not allowed):
+   * Group By Operator:
+   * grouping keys: group by expressions
+   * Mapper: iterate/terminatePartial (mode = HASH)
+   * Partitioning Key: grouping key
+   * Sorting Key: grouping key
+   * Reducer: merge/terminate (mode MERGEPARTIAL)
+   * Group by Operator:
+   * grouping keys: group by expressions + add a new grouping id. key
    *
-   * Sorting Key: grouping key if no DISTINCT grouping + distinct key if
-   * DISTINCT
-   *
-   * Reducer: iterate/terminate if DISTINCT merge/terminate if NO DISTINCT (mode
-   * = MERGEPARTIAL)
+   * STAGE 2
+   * Partitioning Key: grouping key + grouping id.
+   * Sorting Key: grouping key + grouping id.
+   * Reducer: merge/terminate (mode = FINAL)
+   * Group by Operator:
+   * grouping keys: group by expressions + grouping id.
    */
   @SuppressWarnings("nls")
-  private Operator genGroupByPlanMapAggr1MR(String dest, QB qb,
+  private Operator genGroupByPlanMapAggrNoSkew(String dest, QB qb,
       Operator inputOperatorInfo) throws SemanticException {
 
     QBParseInfo parseInfo = qb.getParseInfo();
     ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets =
-      getGroupByGroupingSetsForClause(parseInfo, dest);
+        getGroupByGroupingSetsForClause(parseInfo, dest);
 
     List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst();
     List<Integer> groupingSets = grpByExprsGroupingSets.getSecond();
     boolean groupingSetsPresent = !groupingSets.isEmpty();
 
+    int newMRJobGroupingSetsThreshold =
+        conf.getIntVar(HiveConf.ConfVars.HIVE_NEW_JOB_GROUPING_SET_CARDINALITY);
+
     if (groupingSetsPresent) {
       checkExpressionsForGroupingSet(grpByExprs,
-        parseInfo.getDistinctFuncExprsForClause(dest),
-        parseInfo.getAggregationExprsForClause(dest),
-        opParseCtx.get(inputOperatorInfo).getRowResolver());
+          parseInfo.getDistinctFuncExprsForClause(dest),
+          parseInfo.getAggregationExprsForClause(dest),
+          opParseCtx.get(inputOperatorInfo).getRowResolver());
     }
 
     // ////// Generate GroupbyOperator for a map-side partial aggregation
     Map<String, GenericUDAFEvaluator> genericUDAFEvaluators =
-      new LinkedHashMap<String, GenericUDAFEvaluator>();
+        new LinkedHashMap<String, GenericUDAFEvaluator>();
+
+    // Is the grouping sets data consumed in the current in MR job, or
+    // does it need an additional MR job
+    boolean groupingSetsNeedAdditionalMRJob =
+        groupingSetsPresent && groupingSets.size() > newMRJobGroupingSetsThreshold ?
+            true : false;
+
     GroupByOperator groupByOperatorInfo =
-      (GroupByOperator) genGroupByPlanMapGroupByOperator(
-        qb,
-        dest,
-        grpByExprs,
-        inputOperatorInfo,
-        GroupByDesc.Mode.HASH,
-        genericUDAFEvaluators,
-        groupingSets,
-        groupingSetsPresent);
+        (GroupByOperator) genGroupByPlanMapGroupByOperator(
+            qb,
+            dest,
+            grpByExprs,
+            inputOperatorInfo,
+            GroupByDesc.Mode.HASH,
+            genericUDAFEvaluators,
+            groupingSets,
+            groupingSetsPresent && !groupingSetsNeedAdditionalMRJob);
 
     groupOpToInputTables.put(groupByOperatorInfo, opParseCtx.get(
-      inputOperatorInfo).getRowResolver().getTableNames());
+        inputOperatorInfo).getRowResolver().getTableNames());
     int numReducers = -1;
 
     // Optimize the scenario when there are no grouping keys - only 1 reducer is
@@ -4182,27 +4281,64 @@ public class SemanticAnalyzer extends Ba
     }
 
     // ////// Generate ReduceSink Operator
+    boolean isDistinct = !qb.getParseInfo().getDistinctFuncExprsForClause(dest).isEmpty();
+
+    // Distincts are not allowed with an additional mr job
+    if (groupingSetsNeedAdditionalMRJob && isDistinct) {
+      String errorMsg = "The number of rows per input row due to grouping sets is "
+          + groupingSets.size();
+      throw new SemanticException(
+          ErrorMsg.HIVE_GROUPING_SETS_THRESHOLD_NOT_ALLOWED_WITH_DISTINCTS.getMsg(errorMsg));
+    }
+
     Operator reduceSinkOperatorInfo =
-      genGroupByPlanReduceSinkOperator(qb,
-        dest,
-        groupByOperatorInfo,
-        grpByExprs,
-        grpByExprs.size(),
-        true,
-        numReducers,
-        true,
-        groupingSetsPresent);
-
-    // This is a 1-stage map-reduce processing of the groupby. Tha map-side
-    // aggregates was just used to
-    // reduce output data. In case of distincts, partial results are not used,
-    // and so iterate is again
-    // invoked on the reducer. In case of non-distincts, partial results are
-    // used, and merge is invoked
-    // on the reducer.
-    return genGroupByPlanGroupByOperator1(parseInfo, dest,
-      reduceSinkOperatorInfo, GroupByDesc.Mode.MERGEPARTIAL,
-      genericUDAFEvaluators, false, groupingSetsPresent);
+        genGroupByPlanReduceSinkOperator(qb,
+            dest,
+            groupByOperatorInfo,
+            grpByExprs,
+            grpByExprs.size(),
+            true,
+            numReducers,
+            true,
+            groupingSetsPresent && !groupingSetsNeedAdditionalMRJob);
+
+    // Does it require a new MR job for grouping sets
+    if (!groupingSetsPresent || !groupingSetsNeedAdditionalMRJob) {
+      // This is a 1-stage map-reduce processing of the groupby. Tha map-side
+      // aggregates was just used to
+      // reduce output data. In case of distincts, partial results are not used,
+      // and so iterate is again
+      // invoked on the reducer. In case of non-distincts, partial results are
+      // used, and merge is invoked
+      // on the reducer.
+      return genGroupByPlanGroupByOperator1(parseInfo, dest,
+          reduceSinkOperatorInfo, GroupByDesc.Mode.MERGEPARTIAL,
+          genericUDAFEvaluators, false,
+          groupingSets, groupingSetsPresent, groupingSetsNeedAdditionalMRJob);
+    }
+    else
+    {
+      // Add 'n' rows corresponding to the grouping sets. For each row, create 'n' rows,
+      // one for each grouping set key. Since map-side aggregation has already been performed,
+      // the number of rows would have been reduced. Moreover, the rows corresponding to the
+      // grouping keys come together, so there is a higher chance of finding the rows in the hash
+      // table.
+      Operator groupByOperatorInfo2 =
+          genGroupByPlanGroupByOperator1(parseInfo, dest,
+              reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIALS,
+              genericUDAFEvaluators, false,
+              groupingSets, groupingSetsPresent, groupingSetsNeedAdditionalMRJob);
+
+      // ////// Generate ReduceSinkOperator2
+      Operator reduceSinkOperatorInfo2 = genGroupByPlanReduceSinkOperator2MR(
+          parseInfo, dest, groupByOperatorInfo2, grpByExprs.size() + 1, numReducers,
+          groupingSetsPresent);
+
+      // ////// Generate GroupbyOperator3
+      return genGroupByPlanGroupByOperator2MR(parseInfo, dest,
+          reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL,
+          genericUDAFEvaluators, groupingSetsPresent);
+    }
   }
 
   /**
@@ -4215,25 +4351,48 @@ public class SemanticAnalyzer extends Ba
    * distinct is present) in hope of getting a uniform distribution, and compute
    * partial aggregates grouped by the reduction key (grouping key + distinct
    * key). Evaluate partial aggregates first, and spray by the grouping key to
-   * compute actual aggregates in the second phase. The agggregation evaluation
-   * functions are as follows: Mapper: iterate/terminatePartial (mode = HASH)
-   *
-   * Partitioning Key: random() if no DISTINCT grouping + distinct key if
-   * DISTINCT
+   * compute actual aggregates in the second phase.
    *
-   * Sorting Key: grouping key if no DISTINCT grouping + distinct key if
-   * DISTINCT
+   * The agggregation evaluation functions are as follows:
    *
-   * Reducer: iterate/terminatePartial if DISTINCT merge/terminatePartial if NO
-   * DISTINCT (mode = MERGEPARTIAL)
+   * No grouping sets:
+   * STAGE 1
+   * Group by Operator:
+   * grouping keys: group by expressions if no DISTINCT
+   * grouping keys: group by expressions + distinct keys if DISTINCT
+   * Mapper: iterate/terminatePartial (mode = HASH)
+   * Partitioning Key: random() if no DISTINCT
+   * grouping + distinct key if DISTINCT
+   * Sorting Key: grouping key if no DISTINCT
+   * grouping + distinct key if DISTINCT
+   * Reducer: iterate/terminatePartial if DISTINCT
+   * merge/terminatePartial if NO DISTINCT (mode = MERGEPARTIAL)
+   * Group by Operator:
+   * grouping keys: group by expressions
    *
    * STAGE 2
+   * Partitioning Key: grouping key
+   * Sorting Key: grouping key
+   * Reducer: merge/terminate (mode = FINAL)
    *
-   * Partitioining Key: grouping key
-   *
-   * Sorting Key: grouping key if no DISTINCT grouping + distinct key if
-   * DISTINCT
+   * In the presence of grouping sets, the agggregation evaluation functions are as follows:
+   * STAGE 1
+   * Group by Operator:
+   * grouping keys: group by expressions + grouping id. if no DISTINCT
+   * grouping keys: group by expressions + + grouping id. + distinct keys if DISTINCT
+   * Mapper: iterate/terminatePartial (mode = HASH)
+   * Partitioning Key: random() if no DISTINCT
+   * grouping + grouping id. + distinct key if DISTINCT
+   * Sorting Key: grouping key + grouping id. if no DISTINCT
+   * grouping + grouping id. + distinct key if DISTINCT
+   * Reducer: iterate/terminatePartial if DISTINCT
+   * merge/terminatePartial if NO DISTINCT (mode = MERGEPARTIAL)
+   * Group by Operator:
+   * grouping keys: group by expressions + grouping id.
    *
+   * STAGE 2
+   * Partitioning Key: grouping key
+   * Sorting Key: grouping key + grouping id.
    * Reducer: merge/terminate (mode = FINAL)
    */
   @SuppressWarnings("nls")
@@ -4243,7 +4402,7 @@ public class SemanticAnalyzer extends Ba
     QBParseInfo parseInfo = qb.getParseInfo();
 
     ObjectPair<List<ASTNode>, List<Integer>> grpByExprsGroupingSets =
-      getGroupByGroupingSetsForClause(parseInfo, dest);
+        getGroupByGroupingSetsForClause(parseInfo, dest);
 
     List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst();
     List<Integer> groupingSets = grpByExprsGroupingSets.getSecond();
@@ -4251,18 +4410,29 @@ public class SemanticAnalyzer extends Ba
 
     if (groupingSetsPresent) {
       checkExpressionsForGroupingSet(grpByExprs,
-        parseInfo.getDistinctFuncExprsForClause(dest),
-        parseInfo.getAggregationExprsForClause(dest),
-        opParseCtx.get(inputOperatorInfo).getRowResolver());
+          parseInfo.getDistinctFuncExprsForClause(dest),
+          parseInfo.getAggregationExprsForClause(dest),
+          opParseCtx.get(inputOperatorInfo).getRowResolver());
+
+      int newMRJobGroupingSetsThreshold =
+          conf.getIntVar(HiveConf.ConfVars.HIVE_NEW_JOB_GROUPING_SET_CARDINALITY);
+
+      // Turn off skew if an additional MR job is required anyway for grouping sets.
+      if (groupingSets.size() > newMRJobGroupingSetsThreshold) {
+        String errorMsg = "The number of rows per input row due to grouping sets is "
+            + groupingSets.size();
+        throw new SemanticException(
+            ErrorMsg.HIVE_GROUPING_SETS_THRESHOLD_NOT_ALLOWED_WITH_SKEW.getMsg(errorMsg));
+      }
     }
 
     // ////// Generate GroupbyOperator for a map-side partial aggregation
     Map<String, GenericUDAFEvaluator> genericUDAFEvaluators =
-      new LinkedHashMap<String, GenericUDAFEvaluator>();
+        new LinkedHashMap<String, GenericUDAFEvaluator>();
     GroupByOperator groupByOperatorInfo =
-      (GroupByOperator) genGroupByPlanMapGroupByOperator(
-        qb, dest, grpByExprs, inputOperatorInfo, GroupByDesc.Mode.HASH,
-        genericUDAFEvaluators, groupingSets, groupingSetsPresent);
+        (GroupByOperator) genGroupByPlanMapGroupByOperator(
+            qb, dest, grpByExprs, inputOperatorInfo, GroupByDesc.Mode.HASH,
+            genericUDAFEvaluators, groupingSets, groupingSetsPresent);
 
     groupOpToInputTables.put(groupByOperatorInfo, opParseCtx.get(
         inputOperatorInfo).getRowResolver().getTableNames());
@@ -4274,20 +4444,21 @@ public class SemanticAnalyzer extends Ba
 
       // ////// Generate ReduceSink Operator
       Operator reduceSinkOperatorInfo =
-        genGroupByPlanReduceSinkOperator(qb,
-          dest,
-          groupByOperatorInfo,
-          grpByExprs,
-          distinctFuncExprs.isEmpty() ? -1 : Integer.MAX_VALUE,
-          false,
-          -1,
-          true,
-          groupingSetsPresent);
+          genGroupByPlanReduceSinkOperator(qb,
+              dest,
+              groupByOperatorInfo,
+              grpByExprs,
+              distinctFuncExprs.isEmpty() ? -1 : Integer.MAX_VALUE,
+              false,
+              -1,
+              true,
+              groupingSetsPresent);
 
       // ////// Generate GroupbyOperator for a partial aggregation
       Operator groupByOperatorInfo2 = genGroupByPlanGroupByOperator1(parseInfo,
-        dest, reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIALS,
-        genericUDAFEvaluators, false, groupingSetsPresent);
+          dest, reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIALS,
+          genericUDAFEvaluators, false,
+          groupingSets, groupingSetsPresent, false);
 
       int numReducers = -1;
       if (grpByExprs.isEmpty()) {
@@ -4296,31 +4467,31 @@ public class SemanticAnalyzer extends Ba
 
       // ////// Generate ReduceSinkOperator2
       Operator reduceSinkOperatorInfo2 = genGroupByPlanReduceSinkOperator2MR(
-        parseInfo, dest, groupByOperatorInfo2, grpByExprs.size(), numReducers,
-        groupingSetsPresent);
+          parseInfo, dest, groupByOperatorInfo2, grpByExprs.size(), numReducers,
+          groupingSetsPresent);
 
       // ////// Generate GroupbyOperator3
       return genGroupByPlanGroupByOperator2MR(parseInfo, dest,
-        reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL,
-        genericUDAFEvaluators, groupingSetsPresent);
+          reduceSinkOperatorInfo2, GroupByDesc.Mode.FINAL,
+          genericUDAFEvaluators, groupingSetsPresent);
     } else {
       // If there are no grouping keys, grouping sets cannot be present
       assert !groupingSetsPresent;
 
       // ////// Generate ReduceSink Operator
       Operator reduceSinkOperatorInfo =
-        genGroupByPlanReduceSinkOperator(qb,
-          dest,
-          groupByOperatorInfo,
-          grpByExprs,
-          grpByExprs.size(),
-          false,
-          1,
-          true,
-          groupingSetsPresent);
+          genGroupByPlanReduceSinkOperator(qb,
+              dest,
+              groupByOperatorInfo,
+              grpByExprs,
+              grpByExprs.size(),
+              false,
+              1,
+              true,
+              groupingSetsPresent);
 
       return genGroupByPlanGroupByOperator2MR(parseInfo, dest,
-        reduceSinkOperatorInfo, GroupByDesc.Mode.FINAL, genericUDAFEvaluators, false);
+          reduceSinkOperatorInfo, GroupByDesc.Mode.FINAL, genericUDAFEvaluators, false);
     }
   }
 
@@ -4347,10 +4518,10 @@ public class SemanticAnalyzer extends Ba
   }
 
   private int getReducersBucketing(int totalFiles, int maxReducers) {
-    int numFiles = totalFiles/maxReducers;
+    int numFiles = totalFiles / maxReducers;
     while (true) {
-      if (totalFiles%numFiles == 0) {
-        return totalFiles/numFiles;
+      if (totalFiles % numFiles == 0) {
+        return totalFiles / numFiles;
       }
       numFiles++;
     }
@@ -4359,8 +4530,8 @@ public class SemanticAnalyzer extends Ba
   private static class SortBucketRSCtx {
     ArrayList<ExprNodeDesc> partnCols;
     boolean multiFileSpray;
-    int     numFiles;
-    int     totalFiles;
+    int numFiles;
+    int totalFiles;
 
     public SortBucketRSCtx() {
       partnCols = null;
@@ -4377,7 +4548,8 @@ public class SemanticAnalyzer extends Ba
     }
 
     /**
-     * @param partnCols the partnCols to set
+     * @param partnCols
+     *          the partnCols to set
      */
     public void setPartnCols(ArrayList<ExprNodeDesc> partnCols) {
       this.partnCols = partnCols;
@@ -4391,7 +4563,8 @@ public class SemanticAnalyzer extends Ba
     }
 
     /**
-     * @param multiFileSpray the multiFileSpray to set
+     * @param multiFileSpray
+     *          the multiFileSpray to set
      */
     public void setMultiFileSpray(boolean multiFileSpray) {
       this.multiFileSpray = multiFileSpray;
@@ -4405,7 +4578,8 @@ public class SemanticAnalyzer extends Ba
     }
 
     /**
-     * @param numFiles the numFiles to set
+     * @param numFiles
+     *          the numFiles to set
      */
     public void setNumFiles(int numFiles) {
       this.numFiles = numFiles;
@@ -4419,7 +4593,8 @@ public class SemanticAnalyzer extends Ba
     }
 
     /**
-     * @param totalFiles the totalFiles to set
+     * @param totalFiles
+     *          the totalFiles to set
      */
     public void setTotalFiles(int totalFiles) {
       this.totalFiles = totalFiles;
@@ -4427,9 +4602,8 @@ public class SemanticAnalyzer extends Ba
   }
 
   @SuppressWarnings("nls")
-  private Operator genBucketingSortingDest(String dest, Operator input, QB qb, TableDesc table_desc,
-                                           Table dest_tab, SortBucketRSCtx ctx)
-      throws SemanticException {
+  private Operator genBucketingSortingDest(String dest, Operator input, QB qb,
+      TableDesc table_desc, Table dest_tab, SortBucketRSCtx ctx) throws SemanticException {
 
     // If the table is bucketed, and bucketing is enforced, do the following:
     // If the number of buckets is smaller than the number of maximum reducers,
@@ -4438,20 +4612,21 @@ public class SemanticAnalyzer extends Ba
     // spray the data into multiple buckets. That way, we can support a very large
     // number of buckets without needing a very large number of reducers.
     boolean enforceBucketing = false;
-    boolean enforceSorting   = false;
+    boolean enforceSorting = false;
     ArrayList<ExprNodeDesc> partnCols = new ArrayList<ExprNodeDesc>();
     ArrayList<ExprNodeDesc> partnColsNoConvert = new ArrayList<ExprNodeDesc>();
-    ArrayList<ExprNodeDesc> sortCols  = new ArrayList<ExprNodeDesc>();
+    ArrayList<ExprNodeDesc> sortCols = new ArrayList<ExprNodeDesc>();
     ArrayList<Integer> sortOrders = new ArrayList<Integer>();
     boolean multiFileSpray = false;
-    int     numFiles = 1;
-    int     totalFiles = 1;
+    int numFiles = 1;
+    int totalFiles = 1;
 
     if ((dest_tab.getNumBuckets() > 0) &&
         (conf.getBoolVar(HiveConf.ConfVars.HIVEENFORCEBUCKETING))) {
       enforceBucketing = true;
       partnCols = getParitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, true);
-      partnColsNoConvert = getParitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input, false);
+      partnColsNoConvert = getParitionColsFromBucketCols(dest, qb, dest_tab, table_desc, input,
+          false);
     }
 
     if ((dest_tab.getSortCols() != null) &&
@@ -4471,7 +4646,7 @@ public class SemanticAnalyzer extends Ba
       if (conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS) > 0) {
         maxReducers = conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS);
       }
-      int numBuckets  = dest_tab.getNumBuckets();
+      int numBuckets = dest_tab.getNumBuckets();
       if (numBuckets > maxReducers) {
         multiFileSpray = true;
         totalFiles = numBuckets;
@@ -4481,7 +4656,7 @@ public class SemanticAnalyzer extends Ba
         else {
           // find the number of reducers such that it is a divisor of totalFiles
           maxReducers = getReducersBucketing(totalFiles, maxReducers);
-          numFiles = totalFiles/maxReducers;
+          numFiles = totalFiles / maxReducers;
         }
       }
       else {
@@ -4500,6 +4675,7 @@ public class SemanticAnalyzer extends Ba
 
   /**
    * Check for HOLD_DDLTIME hint.
+   *
    * @param qb
    * @return true if HOLD_DDLTIME is set, false otherwise.
    */
@@ -4525,7 +4701,7 @@ public class SemanticAnalyzer extends Ba
     QBMetaData qbm = qb.getMetaData();
     Integer dest_type = qbm.getDestTypeForAlias(dest);
 
-    Table dest_tab = null;     // destination table if any
+    Table dest_tab = null; // destination table if any
     Partition dest_part = null;// destination partition if any
     String queryTmpdir = null; // the intermediate destination directory
     Path dest_path = null; // the final destination directory
@@ -4546,7 +4722,7 @@ public class SemanticAnalyzer extends Ba
       // Is the user trying to insert into a external tables
       if ((!conf.getBoolVar(HiveConf.ConfVars.HIVE_INSERT_INTO_EXTERNAL_TABLES)) &&
           (dest_tab.getTableType().equals(TableType.EXTERNAL_TABLE))) {
-          throw new SemanticException(
+        throw new SemanticException(
             ErrorMsg.INSERT_EXTERNAL_TABLE.getMsg(dest_tab.getTableName()));
       }
 
@@ -4556,17 +4732,17 @@ public class SemanticAnalyzer extends Ba
       // check for partition
       List<FieldSchema> parts = dest_tab.getPartitionKeys();
       if (parts != null && parts.size() > 0) { // table is partitioned
-        if (partSpec== null || partSpec.size() == 0) { // user did NOT specify partition
+        if (partSpec == null || partSpec.size() == 0) { // user did NOT specify partition
           throw new SemanticException(generateErrorMessage(
-                qb.getParseInfo().getDestForClause(dest),
-                ErrorMsg.NEED_PARTITION_ERROR.getMsg()));
+              qb.getParseInfo().getDestForClause(dest),
+              ErrorMsg.NEED_PARTITION_ERROR.getMsg()));
         }
         // the HOLD_DDLTIIME hint should not be used with dynamic partition since the
         // newly generated partitions should always update their DDLTIME
         if (holdDDLTime) {
           throw new SemanticException(generateErrorMessage(
-                qb.getParseInfo().getDestForClause(dest),
-                ErrorMsg.HOLD_DDLTIME_ON_NONEXIST_PARTITIONS.getMsg()));
+              qb.getParseInfo().getDestForClause(dest),
+              ErrorMsg.HOLD_DDLTIME_ON_NONEXIST_PARTITIONS.getMsg()));
         }
         dpCtx = qbm.getDPCtx(dest);
         if (dpCtx == null) {
@@ -4580,8 +4756,8 @@ public class SemanticAnalyzer extends Ba
         if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONING)) { // allow DP
           if (dpCtx.getNumDPCols() > 0 &&
               (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPFILES) ||
-               HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPREDFILES)) &&
-               Utilities.supportCombineFileInputFormat() == false) {
+              HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPREDFILES)) &&
+              Utilities.supportCombineFileInputFormat() == false) {
             // Do not support merge for Hadoop versions (pre-0.20) that do not
             // support CombineHiveInputFormat
             HiveConf.setBoolVar(conf, HiveConf.ConfVars.HIVEMERGEMAPFILES, false);
@@ -4592,8 +4768,8 @@ public class SemanticAnalyzer extends Ba
 
         } else { // QBMetaData.DEST_PARTITION capture the all-SP case
           throw new SemanticException(generateErrorMessage(
-                qb.getParseInfo().getDestForClause(dest),
-                ErrorMsg.DYNAMIC_PARTITION_DISABLED.getMsg()));
+              qb.getParseInfo().getDestForClause(dest),
+              ErrorMsg.DYNAMIC_PARTITION_DISABLED.getMsg()));
         }
         if (dpCtx.getSPPath() != null) {
           dest_path = new Path(dest_tab.getPath(), dpCtx.getSPPath());
@@ -4663,7 +4839,7 @@ public class SemanticAnalyzer extends Ba
         else {
           try {
             String ppath = dpCtx.getSPPath();
-            ppath = ppath.substring(0, ppath.length()-1);
+            ppath = ppath.substring(0, ppath.length() - 1);
             DummyPartition p =
                 new DummyPartition(dest_tab, dest_tab.getDbName()
                     + "@" + dest_tab.getTableName() + "@" + ppath,
@@ -4683,15 +4859,15 @@ public class SemanticAnalyzer extends Ba
       dest_tab = dest_part.getTable();
       if ((!conf.getBoolVar(HiveConf.ConfVars.HIVE_INSERT_INTO_EXTERNAL_TABLES)) &&
           dest_tab.getTableType().equals(TableType.EXTERNAL_TABLE)) {
-          throw new SemanticException(
+        throw new SemanticException(
             ErrorMsg.INSERT_EXTERNAL_TABLE.getMsg(dest_tab.getTableName()));
       }
 
       Path tabPath = dest_tab.getPath();
       Path partPath = dest_part.getPartitionPath();
 
-        // if the table is in a different dfs than the partition,
-        // replace the partition's dfs with the table's dfs.
+      // if the table is in a different dfs than the partition,
+      // replace the partition's dfs with the table's dfs.
       dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
           .getAuthority(), partPath.toUri().getPath());
 
@@ -4719,8 +4895,8 @@ public class SemanticAnalyzer extends Ba
           Partition part = db.getPartition(dest_tab, dest_part.getSpec(), false);
           if (part == null) {
             throw new SemanticException(generateErrorMessage(
-                  qb.getParseInfo().getDestForClause(dest),
-                  ErrorMsg.HOLD_DDLTIME_ON_NONEXIST_PARTITIONS.getMsg()));
+                qb.getParseInfo().getDestForClause(dest),
+                ErrorMsg.HOLD_DDLTIME_ON_NONEXIST_PARTITIONS.getMsg()));
           }
         } catch (HiveException e) {
           throw new SemanticException(e);
@@ -4920,16 +5096,16 @@ public class SemanticAnalyzer extends Ba
     }
 
     Operator output = putOpInsertMap(OperatorFactory.getAndMakeChild(fileSinkDesc,
-            fsRS, input), inputRR);
+        fsRS, input), inputRR);
 
     if (ltd != null && SessionState.get() != null) {
       SessionState.get().getLineageState()
-        .mapDirToFop(ltd.getSourceDir(), (FileSinkOperator)output);
+          .mapDirToFop(ltd.getSourceDir(), (FileSinkOperator) output);
     }
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("Created FileSink Plan for clause: " + dest + "dest_path: "
-                + dest_path + " row schema: " + inputRR.toString());
+          + dest_path + " row schema: " + inputRR.toString());
     }
 
     return output;
@@ -4959,7 +5135,7 @@ public class SemanticAnalyzer extends Ba
     int inColumnCnt = rowFields.size();
     int outColumnCnt = tableFields.size();
     if (dynPart && dpCtx != null) {
-        outColumnCnt += dpCtx.getNumDPCols();
+      outColumnCnt += dpCtx.getNumDPCols();
     }
 
     if (inColumnCnt != outColumnCnt) {
@@ -4967,7 +5143,7 @@ public class SemanticAnalyzer extends Ba
           + " columns, but query has " + inColumnCnt + " columns.";
       throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg(
           qb.getParseInfo().getDestForClause(dest), reason));
-    } else if (dynPart && dpCtx != null){
+    } else if (dynPart && dpCtx != null) {
       // create the mapping from input ExprNode to dest table DP column
       dpCtx.mapInputToDP(rowFields.subList(tableFields.size(), rowFields.size()));
     }
@@ -4998,8 +5174,8 @@ public class SemanticAnalyzer extends Ba
         // JSON-format.
         if (!tableFieldTypeInfo.equals(rowFieldTypeInfo)
             && !(isLazySimpleSerDe
-            && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE) && tableFieldTypeInfo
-            .equals(TypeInfoFactory.stringTypeInfo))) {
+                && tableFieldTypeInfo.getCategory().equals(Category.PRIMITIVE) && tableFieldTypeInfo
+                  .equals(TypeInfoFactory.stringTypeInfo))) {
           // need to do some conversions here
           converted = true;
           if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) {
@@ -5024,7 +5200,7 @@ public class SemanticAnalyzer extends Ba
     // deal with dynamic partition columns: convert ExprNodeDesc type to String??
     if (dynPart && dpCtx != null && dpCtx.getNumDPCols() > 0) {
       // DP columns starts with tableFields.size()

[... 1192 lines stripped ...]