svn commit: r1667139 - in /hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql: optimizer/calcite/reloperators/HiveTableScan.java parse/CalcitePlanner.java

[jp...@apache.org]

Author: jpullokk
Date: Mon Mar 16 21:53:37 2015
New Revision: 1667139

URL: http://svn.apache.org/r1667139
Log:
HIVE-9982:CBO (Calcite Return Path) Prune TS Relnode schema (Laljo John Pullokkaran)

Modified:
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java?rev=1667139&r1=1667138&r2=1667139&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableScan.java Mon Mar 16 21:53:37 2015
@@ -17,15 +17,21 @@
  */
 package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
 
+import java.util.LinkedList;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.calcite.plan.RelOptCluster;
 import org.apache.calcite.plan.RelOptCost;
 import org.apache.calcite.plan.RelOptPlanner;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.RelFactories;
 import org.apache.calcite.rel.core.TableScan;
 import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
 import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveCost;
@@ -100,4 +106,35 @@ public class HiveTableScan extends Table
     return ((RelOptHiveTable) table).getColStat(projIndxLst);
   }
 
+  @Override
+  public RelNode project(ImmutableBitSet fieldsUsed, Set<RelDataTypeField> extraFields,
+      RelFactories.ProjectFactory projectFactory) {
+
+    // 1. If the schema is the same then bail out
+    final int fieldCount = getRowType().getFieldCount();
+    if (fieldsUsed.equals(ImmutableBitSet.range(fieldCount)) && extraFields.isEmpty()) {
+      return this;
+    }
+
+    // 2. Make sure there is no dynamic addition of virtual cols
+    if (extraFields != null && !extraFields.isEmpty()) {
+      throw new RuntimeException("Hive TS does not support adding virtual columns dynamically");
+    }
+
+    // 3. Create new TS schema that is a subset of original
+    final List<RelDataTypeField> fields = getRowType().getFieldList();
+    List<RelDataType> fieldTypes = new LinkedList<RelDataType>();
+    List<String> fieldNames = new LinkedList<String>();
+    for (int i : fieldsUsed) {
+      RelDataTypeField field = fields.get(i);
+      fieldTypes.add(field.getType());
+      fieldNames.add(field.getName());
+    }
+
+    // 4. Build new TS
+    HiveTableScan newHT = copy(getCluster().getTypeFactory().createStructType(fieldTypes,
+        fieldNames));
+
+    return newHT;
+  }
 }
\ No newline at end of file

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1667139&r1=1667138&r2=1667139&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Mon Mar 16 21:53:37 2015
@@ -58,8 +58,10 @@ import org.apache.calcite.rel.core.Aggre
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Join;
 import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.core.RelFactories;
 import org.apache.calcite.rel.core.SemiJoin;
+import org.apache.calcite.rel.core.Sort;
 import org.apache.calcite.rel.metadata.CachingRelMetadataProvider;
 import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider;
 import org.apache.calcite.rel.metadata.RelMetadataProvider;
@@ -606,13 +608,39 @@ public class CalcitePlanner extends Sema
       throw new AssertionError("rethrowCalciteException didn't throw for " + e.getMessage());
     }
 
+    RelNode modifiedOptimizedOptiqPlan = introduceProjectIfNeeded(optimizedOptiqPlan);
+
     Operator hiveRoot = new HiveOpConverter(topOps, HiveOpConverter.getAggOPMode(conf),
-        conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")).convert(optimizedOptiqPlan);
+        conf.getVar(HiveConf.ConfVars.HIVEMAPREDMODE).equalsIgnoreCase("strict")).convert(modifiedOptimizedOptiqPlan);
     RowResolver hiveRootRR = genRowResolver(hiveRoot, getQB());
     opParseCtx.put(hiveRoot, new OpParseContext(hiveRootRR));
     return genFileSinkPlan(getQB().getParseInfo().getClauseNames().iterator().next(), getQB(), hiveRoot);
   }
 
+  private RelNode introduceProjectIfNeeded(RelNode optimizedOptiqPlan)
+      throws CalciteSemanticException {
+    RelNode parent = null;
+    RelNode input = optimizedOptiqPlan;
+    RelNode newRoot = optimizedOptiqPlan;
+
+    while (!(input instanceof Project) && (input instanceof Sort)) {
+      parent = input;
+      input = input.getInput(0);
+    }
+
+    if (!(input instanceof Project)) {
+      HiveProject hpRel = HiveProject.create(input,
+          HiveCalciteUtil.getProjsFromBelowAsInputRef(input), input.getRowType().getFieldNames());
+      if (input == optimizedOptiqPlan) {
+        newRoot = hpRel;
+      } else {
+        parent.replaceInput(0, hpRel);
+      }
+    }
+
+    return newRoot;
+  }
+
   /***
    * Unwraps Calcite Invocation exceptions coming meta data provider chain and
    * obtains the real cause.